mirror of https://github.com/scrapy/scrapy.git
Merge f0cc534737 into dd10cb8e9a
This commit is contained in:
Adrián Chaves
GitHub
commit
ff031ab186
10
docs/conf.py
10
docs/conf.py
|
|
@ -31,6 +31,7 @@ extensions = [
|
|||
"sphinx_scrapy",
|
||||
"scrapyfixautodoc", # Must be after "sphinx.ext.autodoc"
|
||||
"sphinx.ext.coverage",
|
||||
"sphinx_reredirects",
|
||||
"sphinx_rtd_dark_mode",
|
||||
]
|
||||
|
||||
|
|
@ -141,12 +142,21 @@ coverage_ignore_pyobjects = [
|
|||
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
|
||||
]
|
||||
|
||||
# -- Options for the autodoc extension ----------------------------------------
|
||||
autodoc_type_aliases = {
|
||||
"RequestScopes": "RequestScopes",
|
||||
}
|
||||
|
||||
# -- Options for the InterSphinx extension -----------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
|
||||
|
||||
intersphinx_disabled_reftypes: Sequence[str] = []
|
||||
|
||||
# -- sphinx-reredirects -------------------------------------------------------
|
||||
redirects = {
|
||||
"topics/autothrottle": "throttling.html",
|
||||
}
|
||||
|
||||
# sphinx-scrapy ---------------------------------------------------------------
|
||||
|
||||
scrapy_intersphinx_enable = [
|
||||
|
|
|
|||
|
|
@ -158,7 +158,7 @@ Solving specific problems
|
|||
topics/leaks
|
||||
topics/media-pipeline
|
||||
topics/deploy
|
||||
topics/autothrottle
|
||||
topics/throttling
|
||||
topics/benchmarking
|
||||
topics/jobs
|
||||
topics/coroutines
|
||||
|
|
@ -198,8 +198,9 @@ Solving specific problems
|
|||
:doc:`topics/deploy`
|
||||
Deploying your Scrapy spiders and run them in a remote server.
|
||||
|
||||
:doc:`topics/autothrottle`
|
||||
Adjust crawl rate dynamically based on load.
|
||||
:doc:`topics/throttling`
|
||||
Control request throttling to avoid overloading websites and comply with
|
||||
rate limits.
|
||||
|
||||
:doc:`topics/benchmarking`
|
||||
Check how Scrapy performs on your hardware.
|
||||
|
|
|
|||
|
|
@ -81,11 +81,9 @@ error happens while handling it.
|
|||
|
||||
While this enables you to do very fast crawls (sending multiple concurrent
|
||||
requests at the same time, in a fault-tolerant way) Scrapy also gives you
|
||||
control over the politeness of the crawl through :ref:`a few settings
|
||||
<topics-settings-ref>`. You can do things like setting a download delay between
|
||||
each request, limiting the amount of concurrent requests per domain or per IP, and
|
||||
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
||||
to figure these settings out automatically.
|
||||
control over :ref:`throttling <throttling>`, e.g. you can set a delay between
|
||||
requests to the same domain, set a maximum concurrency per domain, or customize
|
||||
the :ref:`backoff <backoff>` behavior.
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
|
|||
|
|
@ -130,6 +130,28 @@ and defines some attributes and methods:
|
|||
the scraped data as dicts and also finding new URLs to
|
||||
follow and creating new requests (:class:`~scrapy.Request`) from them.
|
||||
|
||||
Crawling faster for this tutorial
|
||||
---------------------------------
|
||||
|
||||
By default, Scrapy :ref:`throttles <throttling>` crawls to be polite: it sends
|
||||
at most one request at a time to each website, waiting one second between
|
||||
requests. That keeps you from overwhelming websites, but it also makes crawling
|
||||
slow, and it means the requests above are sent one after another rather than in
|
||||
parallel.
|
||||
|
||||
``quotes.toscrape.com`` is a sandbox meant for scraping practice, so we can
|
||||
safely crawl it faster. Open ``tutorial/settings.py`` and add a
|
||||
:setting:`THROTTLER_SCOPES` entry that raises the concurrency and lowers the
|
||||
delay for that domain only:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
THROTTLER_SCOPES = {
|
||||
"quotes.toscrape.com": {"concurrency": 16, "delay": 0.1},
|
||||
}
|
||||
|
||||
The polite defaults still apply to every other website.
|
||||
|
||||
How to run our spider
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -1173,7 +1173,7 @@ Deprecations
|
|||
(:issue:`7005`, :issue:`7043`)
|
||||
|
||||
- The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead.
|
||||
``CONCURRENT_REQUESTS_PER_DOMAIN`` instead.
|
||||
(:issue:`6917`, :issue:`6921`)
|
||||
|
||||
- The ``scrapy.core.downloader.handlers.http`` module is deprecated. You
|
||||
|
|
@ -1505,7 +1505,7 @@ Scrapy 2.13.3 (2025-07-02)
|
|||
--------------------------
|
||||
|
||||
- Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the
|
||||
``CONCURRENT_REQUESTS_PER_DOMAIN`` (from ``8`` to ``1``) in the
|
||||
default project template.
|
||||
(:issue:`6597`, :issue:`6918`, :issue:`6923`)
|
||||
|
||||
|
|
@ -2491,10 +2491,9 @@ New features
|
|||
:meth:`~scrapy.crawler.Crawler.get_spider_middleware`.
|
||||
(:issue:`6181`)
|
||||
|
||||
- Slot delay updates by the :ref:`AutoThrottle extension
|
||||
<topics-autothrottle>` based on response latencies can now be disabled for
|
||||
specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta
|
||||
key.
|
||||
- Slot delay updates by the ``scrapy.extensions.throttle.AutoThrottle``
|
||||
extension based on response latencies can now be disabled for specific
|
||||
requests via the ``autothrottle_dont_adjust_delay`` meta key.
|
||||
(:issue:`6246`, :issue:`6527`)
|
||||
|
||||
- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``,
|
||||
|
|
@ -3260,9 +3259,9 @@ New features
|
|||
~~~~~~~~~~~~
|
||||
|
||||
- Settings corresponding to :setting:`DOWNLOAD_DELAY`,
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
``CONCURRENT_REQUESTS_PER_DOMAIN`` and
|
||||
:setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
|
||||
via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
|
||||
via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`)
|
||||
|
||||
- Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors
|
||||
available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`)
|
||||
|
|
@ -6276,7 +6275,7 @@ New features
|
|||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||
scheduling improvement on crawls targeting multiple web domains, at the
|
||||
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||
cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
|
||||
|
||||
* A new :attr:`.Request.cb_kwargs` attribute
|
||||
provides a cleaner way to pass keyword arguments to callback methods
|
||||
|
|
@ -6476,8 +6475,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
|
|||
|
||||
* From :class:`~scrapy.spiders.Spider` (and subclasses):
|
||||
|
||||
* ``DOWNLOAD_DELAY`` (use :ref:`download_delay
|
||||
<spider-download_delay-attribute>`)
|
||||
* ``DOWNLOAD_DELAY`` (use ``download_delay``)
|
||||
|
||||
* ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`)
|
||||
|
||||
|
|
@ -7450,7 +7448,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
|
|||
|
||||
- Item loaders now support nested loaders (:issue:`1467`).
|
||||
- ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`).
|
||||
- Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved
|
||||
- Added setting ``AUTOTHROTTLE_TARGET_CONCURRENCY`` and improved
|
||||
AutoThrottle docs (:issue:`1324`).
|
||||
- Added ``response.text`` to get body as unicode (:issue:`1730`).
|
||||
- Anonymous S3 connections (:issue:`1358`).
|
||||
|
|
@ -8653,7 +8651,7 @@ Scrapy changes:
|
|||
|
||||
- added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way
|
||||
- added options ``-o`` and ``-t`` to the :command:`runspider` command
|
||||
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
|
||||
- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with ``AUTOTHROTTLE_ENABLED``
|
||||
- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
|
||||
- added a ``process_start_requests()`` method to spider middlewares
|
||||
- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
|
|
@ -8755,7 +8753,7 @@ New features and settings
|
|||
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
|
||||
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
||||
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||
- :setting:`CONCURRENT_REQUESTS`, ``CONCURRENT_REQUESTS_PER_DOMAIN``, ``CONCURRENT_REQUESTS_PER_IP``
|
||||
- check the documentation for more details
|
||||
- Added builtin caching DNS resolver (:rev:`2728`)
|
||||
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ pydantic
|
|||
scrapy-spider-metadata
|
||||
sphinx
|
||||
sphinx-notfound-page
|
||||
sphinx-reredirects
|
||||
sphinx-rtd-theme
|
||||
sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ sphinx==9.1.0
|
|||
# sphinx-llms-txt
|
||||
# sphinx-markdown-builder
|
||||
# sphinx-notfound-page
|
||||
# sphinx-reredirects
|
||||
# sphinx-rtd-theme
|
||||
# sphinx-scrapy
|
||||
# sphinxcontrib-jquery
|
||||
|
|
@ -147,6 +148,8 @@ sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builde
|
|||
# via sphinx-scrapy
|
||||
sphinx-notfound-page==1.1.0
|
||||
# via -r docs/requirements.in
|
||||
sphinx-reredirects==1.1.0
|
||||
# via -r docs/requirements.in
|
||||
sphinx-rtd-dark-mode==1.3.0
|
||||
# via -r docs/requirements.in
|
||||
sphinx-rtd-theme==3.1.0
|
||||
|
|
|
|||
|
|
@ -93,6 +93,8 @@ how you :ref:`configure the downloader middlewares
|
|||
modify the downloader and scheduler behaviour, although this is an
|
||||
advanced use and this API is not yet stable.
|
||||
|
||||
.. autoattribute:: throttler
|
||||
|
||||
.. attribute:: spider
|
||||
|
||||
Spider currently being crawled. This is an instance of the spider class
|
||||
|
|
|
|||
|
|
@ -1,190 +0,0 @@
|
|||
.. _topics-autothrottle:
|
||||
|
||||
======================
|
||||
AutoThrottle extension
|
||||
======================
|
||||
|
||||
This is an extension for automatically throttling crawling speed based on load
|
||||
of both the Scrapy server and the website you are crawling.
|
||||
|
||||
Design goals
|
||||
============
|
||||
|
||||
1. be nicer to sites instead of using default download delay of zero
|
||||
2. automatically adjust Scrapy to the optimum crawling speed, so the user
|
||||
doesn't have to tune the download delays to find the optimum one.
|
||||
The user only needs to specify the maximum concurrent requests
|
||||
it allows, and the extension does the rest.
|
||||
|
||||
.. _autothrottle-algorithm:
|
||||
|
||||
How it works
|
||||
============
|
||||
|
||||
Scrapy allows defining the concurrency and delay of different download slots,
|
||||
e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are
|
||||
assigned to slots based on their URL domain, although it is possible to
|
||||
customize the download slot of any request.
|
||||
|
||||
The AutoThrottle extension adjusts the delay of each download slot dynamically,
|
||||
to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent
|
||||
requests on average to each remote website.
|
||||
|
||||
It uses download latency to compute the delays. The main idea is the
|
||||
following: if a server needs ``latency`` seconds to respond, a client
|
||||
should send a request each ``latency/N`` seconds to have ``N`` requests
|
||||
processed in parallel.
|
||||
|
||||
Instead of adjusting the delays one can just set a small fixed
|
||||
download delay and impose hard limits on concurrency using
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar
|
||||
effect, but there are some important differences:
|
||||
|
||||
* because the download delay is small there will be occasional bursts
|
||||
of requests;
|
||||
* often non-200 (error) responses can be returned faster than regular
|
||||
responses, so with a small download delay and a hard concurrency limit
|
||||
crawler will be sending requests to server faster when server starts to
|
||||
return errors. But this is an opposite of what crawler should do - in case
|
||||
of errors it makes more sense to slow down: these errors may be caused by
|
||||
the high request rate.
|
||||
|
||||
AutoThrottle doesn't have these issues.
|
||||
|
||||
Throttling algorithm
|
||||
====================
|
||||
|
||||
AutoThrottle algorithm adjusts download delays based on the following rules:
|
||||
|
||||
1. spiders always start with a download delay of
|
||||
:setting:`AUTOTHROTTLE_START_DELAY`;
|
||||
2. when a response is received, the target download delay is calculated as
|
||||
``latency / N`` where ``latency`` is a latency of the response,
|
||||
and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`.
|
||||
3. download delay for next requests is set to the average of previous
|
||||
download delay and the target download delay;
|
||||
4. latencies of non-200 responses are not allowed to decrease the delay;
|
||||
5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater
|
||||
than :setting:`AUTOTHROTTLE_MAX_DELAY`
|
||||
|
||||
.. note:: The AutoThrottle extension honours the standard Scrapy settings for
|
||||
concurrency and delay. This means that it will respect
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
never set a download delay lower than :setting:`DOWNLOAD_DELAY`.
|
||||
|
||||
.. _download-latency:
|
||||
|
||||
In Scrapy, the download latency is measured as the time elapsed between
|
||||
establishing the TCP connection and receiving the HTTP headers.
|
||||
|
||||
Note that these latencies are very hard to measure accurately in a cooperative
|
||||
multitasking environment because Scrapy may be busy processing a spider
|
||||
callback, for example, and unable to attend downloads. However, these latencies
|
||||
should still give a reasonable estimate of how busy Scrapy (and ultimately, the
|
||||
server) is, and this extension builds on that premise.
|
||||
|
||||
.. reqmeta:: autothrottle_dont_adjust_delay
|
||||
|
||||
Prevent specific requests from triggering slot delay adjustments
|
||||
================================================================
|
||||
|
||||
AutoThrottle adjusts the delay of download slots based on the latencies of
|
||||
responses that belong to that download slot. The only exceptions are non-200
|
||||
responses, which are only taken into account to increase that delay, but
|
||||
ignored if they would decrease that delay.
|
||||
|
||||
You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to
|
||||
``True`` in any request to prevent its response latency from impacting the
|
||||
delay of its download slot:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Request
|
||||
|
||||
Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True})
|
||||
|
||||
Note, however, that AutoThrottle still determines the starting delay of every
|
||||
download slot by setting the ``download_delay`` attribute on the running
|
||||
spider. If you want AutoThrottle not to impact a download slot at all, in
|
||||
addition to setting this meta key in all requests that use that download slot,
|
||||
you might want to set a custom value for the ``delay`` attribute of that
|
||||
download slot, e.g. using :setting:`DOWNLOAD_SLOTS`.
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
The settings used to control the AutoThrottle extension are:
|
||||
|
||||
* :setting:`AUTOTHROTTLE_ENABLED`
|
||||
* :setting:`AUTOTHROTTLE_START_DELAY`
|
||||
* :setting:`AUTOTHROTTLE_MAX_DELAY`
|
||||
* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
|
||||
* :setting:`AUTOTHROTTLE_DEBUG`
|
||||
* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
* :setting:`DOWNLOAD_DELAY`
|
||||
|
||||
For more information see :ref:`autothrottle-algorithm`.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_ENABLED
|
||||
|
||||
AUTOTHROTTLE_ENABLED
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Enables the AutoThrottle extension.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_START_DELAY
|
||||
|
||||
AUTOTHROTTLE_START_DELAY
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``5.0``
|
||||
|
||||
The initial download delay (in seconds).
|
||||
|
||||
.. setting:: AUTOTHROTTLE_MAX_DELAY
|
||||
|
||||
AUTOTHROTTLE_MAX_DELAY
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``60.0``
|
||||
|
||||
The maximum download delay (in seconds) to be set in case of high latencies.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY
|
||||
|
||||
AUTOTHROTTLE_TARGET_CONCURRENCY
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``1.0``
|
||||
|
||||
Average number of requests Scrapy should be sending in parallel to remote
|
||||
websites. It must be higher than ``0.0``.
|
||||
|
||||
By default, AutoThrottle adjusts the delay to send a single
|
||||
concurrent request to each of the remote websites. Set this option to
|
||||
a higher value (e.g. ``2.0``) to increase the throughput and the load on remote
|
||||
servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value
|
||||
(e.g. ``0.5``) makes the crawler more conservative and polite.
|
||||
|
||||
Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected
|
||||
when AutoThrottle extension is enabled. This means that if
|
||||
``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number
|
||||
of concurrent requests.
|
||||
|
||||
At every given time point Scrapy can be sending more or less concurrent
|
||||
requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested
|
||||
value the crawler tries to approach, not a hard limit.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_DEBUG
|
||||
|
||||
AUTOTHROTTLE_DEBUG
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Enable AutoThrottle debug mode which will display stats on every response
|
||||
received, so you can see how the throttling parameters are being adjusted in
|
||||
real time.
|
||||
|
|
@ -47,8 +47,8 @@ Increase concurrency
|
|||
====================
|
||||
|
||||
Concurrency is the number of requests that are processed in parallel. There is
|
||||
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
|
||||
can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).
|
||||
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional per-scope
|
||||
(per-domain by default) limit (:setting:`THROTTLER_SCOPE_CONCURRENCY`).
|
||||
|
||||
The default global concurrency limit in Scrapy is not suitable for crawling
|
||||
many different domains in parallel, so you will want to increase it. How much
|
||||
|
|
|
|||
|
|
@ -9,37 +9,24 @@ A Scrapy component is any class whose objects are built using
|
|||
|
||||
That includes the classes that you may assign to the following settings:
|
||||
|
||||
- :setting:`ADDONS`
|
||||
|
||||
- :setting:`TWISTED_DNS_RESOLVER`
|
||||
|
||||
- :setting:`DOWNLOAD_HANDLERS`
|
||||
|
||||
- :setting:`DOWNLOADER_MIDDLEWARES`
|
||||
|
||||
- :setting:`DUPEFILTER_CLASS`
|
||||
|
||||
- :setting:`EXTENSIONS`
|
||||
|
||||
- :setting:`FEED_EXPORTERS`
|
||||
|
||||
- :setting:`FEED_STORAGES`
|
||||
|
||||
- :setting:`ITEM_PIPELINES`
|
||||
|
||||
- :setting:`SCHEDULER`
|
||||
|
||||
- :setting:`SCHEDULER_DISK_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_MEMORY_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_PRIORITY_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_START_DISK_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
|
||||
|
||||
- :setting:`SPIDER_MIDDLEWARES`
|
||||
- :setting:`ADDONS`
|
||||
- :setting:`DOWNLOAD_HANDLERS`
|
||||
- :setting:`DOWNLOADER_MIDDLEWARES`
|
||||
- :setting:`DUPEFILTER_CLASS`
|
||||
- :setting:`EXTENSIONS`
|
||||
- :setting:`FEED_EXPORTERS`
|
||||
- :setting:`FEED_STORAGES`
|
||||
- :setting:`ITEM_PIPELINES`
|
||||
- :setting:`SCHEDULER`
|
||||
- :setting:`SCHEDULER_DISK_QUEUE`
|
||||
- :setting:`SCHEDULER_MEMORY_QUEUE`
|
||||
- :setting:`SCHEDULER_PRIORITY_QUEUE`
|
||||
- :setting:`SCHEDULER_START_DISK_QUEUE`
|
||||
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
|
||||
- :setting:`SPIDER_MIDDLEWARES`
|
||||
- :setting:`THROTTLER`
|
||||
- :setting:`THROTTLER_SCOPE_MANAGER`
|
||||
- :setting:`TWISTED_DNS_RESOLVER`
|
||||
|
||||
Third-party Scrapy components may also let you define additional Scrapy
|
||||
components, usually configurable through :ref:`settings <topics-settings>`, to
|
||||
|
|
|
|||
|
|
@ -169,6 +169,13 @@ middleware, see the :ref:`downloader middleware usage guide
|
|||
For a list of the components enabled by default (and their orders) see the
|
||||
:setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
|
||||
|
||||
.. _backoff-mw:
|
||||
|
||||
BackoffMiddleware
|
||||
-----------------
|
||||
|
||||
.. autoclass:: scrapy.downloadermiddlewares.backoff.BackoffMiddleware
|
||||
|
||||
.. _cookies-mw:
|
||||
|
||||
CookiesMiddleware
|
||||
|
|
@ -942,6 +949,22 @@ Default: ``20``
|
|||
The maximum number of redirections that will be followed for a single request.
|
||||
If maximum redirections are exceeded, the request is aborted and ignored.
|
||||
|
||||
.. setting:: REDIRECT_MAX_DELAY
|
||||
|
||||
REDIRECT_MAX_DELAY
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``120.0``
|
||||
|
||||
If a redirect response provides a ``Retry-After`` header, the redirect is only
|
||||
followed after the specified delay. This setting caps that delay to a specific
|
||||
number of seconds. Set to ``0.0`` to disable these delays altogether.
|
||||
|
||||
Note that redirect delays only affect the request being redirected, they do not
|
||||
delay other requests to the same domain. ``Retry-After`` only does that during
|
||||
:ref:`backoff <retry-after>`.
|
||||
|
||||
|
||||
MetaRefreshMiddleware
|
||||
---------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -706,7 +706,6 @@ are some special keys recognized by Scrapy and its built-in extensions.
|
|||
Those are:
|
||||
|
||||
* :reqmeta:`allow_offsite`
|
||||
* :reqmeta:`autothrottle_dont_adjust_delay`
|
||||
* :reqmeta:`bindaddress`
|
||||
* :reqmeta:`cookiejar`
|
||||
* :reqmeta:`dont_cache`
|
||||
|
|
@ -714,6 +713,7 @@ Those are:
|
|||
* :reqmeta:`dont_obey_robotstxt`
|
||||
* :reqmeta:`dont_redirect`
|
||||
* :reqmeta:`dont_retry`
|
||||
* :reqmeta:`dont_throttle`
|
||||
* :reqmeta:`download_fail_on_dataloss`
|
||||
* :reqmeta:`download_latency`
|
||||
* :reqmeta:`download_maxsize`
|
||||
|
|
@ -733,6 +733,8 @@ Those are:
|
|||
* :reqmeta:`redirect_reasons`
|
||||
* :reqmeta:`redirect_urls`
|
||||
* :reqmeta:`referrer_policy`
|
||||
* :reqmeta:`delay`
|
||||
* :reqmeta:`throttler_scopes`
|
||||
* :reqmeta:`verbatim_url`
|
||||
|
||||
.. reqmeta:: bindaddress
|
||||
|
|
|
|||
|
|
@ -538,21 +538,9 @@ CONCURRENT_REQUESTS
|
|||
|
||||
Default: ``16``
|
||||
|
||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||
performed by the Scrapy downloader.
|
||||
Maximum number of total concurrent requests allowed.
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
------------------------------
|
||||
|
||||
Default: ``1`` (:ref:`fallback <default-settings>`: ``8``)
|
||||
|
||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||
performed to any single domain.
|
||||
|
||||
See also: :ref:`topics-autothrottle` and its
|
||||
:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
|
||||
.. seealso:: :ref:`throttling`
|
||||
|
||||
|
||||
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
|
||||
|
|
@ -873,6 +861,7 @@ Default:
|
|||
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
|
||||
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
|
||||
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
|
||||
"scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 650,
|
||||
"scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700,
|
||||
"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
|
||||
"scrapy.downloadermiddlewares.stats.DownloaderStats": 850,
|
||||
|
|
@ -894,45 +883,6 @@ Default: ``True``
|
|||
|
||||
Whether to enable downloader stats collection.
|
||||
|
||||
.. setting:: DOWNLOAD_DELAY
|
||||
|
||||
DOWNLOAD_DELAY
|
||||
--------------
|
||||
|
||||
Default: ``1`` (:ref:`fallback <default-settings>`: ``0``)
|
||||
|
||||
Minimum seconds to wait between 2 consecutive requests to the same domain.
|
||||
|
||||
Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
|
||||
servers too hard.
|
||||
|
||||
Decimal numbers are supported. For example, to send a maximum of 4 requests
|
||||
every 10 seconds::
|
||||
|
||||
DOWNLOAD_DELAY = 2.5
|
||||
|
||||
This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
|
||||
setting, which is enabled by default.
|
||||
|
||||
Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
|
||||
concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
|
||||
time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
|
||||
concurrency for that domain is 1. When testing throttling configurations, it
|
||||
usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first,
|
||||
and only increase :setting:`DOWNLOAD_DELAY` once
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is
|
||||
desired.
|
||||
|
||||
.. _spider-download_delay-attribute:
|
||||
|
||||
.. note::
|
||||
|
||||
This delay can be set per spider using :attr:`download_delay` spider attribute.
|
||||
|
||||
It is also possible to change this setting per domain, although it requires
|
||||
non-trivial code. See the implementation of the :ref:`AutoThrottle
|
||||
<topics-autothrottle>` extension for an example.
|
||||
|
||||
.. setting:: DOWNLOAD_BIND_ADDRESS
|
||||
|
||||
DOWNLOAD_BIND_ADDRESS
|
||||
|
|
@ -1039,30 +989,6 @@ handler (without replacement), place this in your ``settings.py``:
|
|||
:ref:`security-local-resources`
|
||||
|
||||
|
||||
.. setting:: DOWNLOAD_SLOTS
|
||||
|
||||
DOWNLOAD_SLOTS
|
||||
--------------
|
||||
|
||||
Default: ``{}``
|
||||
|
||||
Allows to define concurrency/delay parameters on per slot (domain) basis:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
DOWNLOAD_SLOTS = {
|
||||
"quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
|
||||
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
|
||||
}
|
||||
|
||||
.. note::
|
||||
|
||||
For other downloader slots default settings values will be used:
|
||||
|
||||
- :setting:`DOWNLOAD_DELAY`: ``delay``
|
||||
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
|
||||
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
|
||||
|
||||
|
||||
.. setting:: DOWNLOAD_TIMEOUT
|
||||
|
||||
|
|
@ -1733,26 +1659,6 @@ Example::
|
|||
|
||||
NEWSPIDER_MODULE = 'mybot.spiders_dev'
|
||||
|
||||
.. setting:: RANDOMIZE_DOWNLOAD_DELAY
|
||||
|
||||
RANDOMIZE_DOWNLOAD_DELAY
|
||||
------------------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
If enabled, Scrapy will wait a random amount of time (between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`) while fetching requests from the same
|
||||
website.
|
||||
|
||||
This randomization decreases the chance of the crawler being detected (and
|
||||
subsequently blocked) by sites which analyze requests looking for statistically
|
||||
significant similarities in the time between their requests.
|
||||
|
||||
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
||||
|
||||
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||
|
||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||
|
||||
.. setting:: REACTOR_THREADPOOL_MAXSIZE
|
||||
|
||||
REACTOR_THREADPOOL_MAXSIZE
|
||||
|
|
|
|||
|
|
@ -502,6 +502,25 @@ headers_received
|
|||
:param spider: the spider associated with the response
|
||||
:type spider: :class:`~scrapy.Spider` object
|
||||
|
||||
robots_parsed
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
.. signal:: robots_parsed
|
||||
.. function:: robots_parsed(robotparser, request)
|
||||
|
||||
Sent by
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
|
||||
downloads and parses a ``robots.txt`` file, for the host that *request*
|
||||
targets.
|
||||
|
||||
This signal supports :ref:`asynchronous handlers <signal-deferred>`.
|
||||
|
||||
:param robotparser: the parser holding the parsed ``robots.txt`` contents
|
||||
:type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
|
||||
|
||||
:param request: the request that triggered the ``robots.txt`` download
|
||||
:type request: :class:`~scrapy.Request` object
|
||||
|
||||
|
||||
Response signals
|
||||
----------------
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -18,7 +18,7 @@ dependencies = [
|
|||
"parsel>=1.5.0",
|
||||
"protego>=0.1.15",
|
||||
"pyOpenSSL>=22.0.0",
|
||||
"queuelib>=1.4.2",
|
||||
"queuelib>=1.6.1",
|
||||
"service_identity>=23.1.0",
|
||||
"tldextract",
|
||||
"w3lib>=1.17.0",
|
||||
|
|
|
|||
|
|
@ -1,48 +1,39 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import warnings
|
||||
from collections import deque
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
|
||||
from scrapy.resolver import dnscache
|
||||
from scrapy.utils.asyncio import (
|
||||
AsyncioLoopingCall,
|
||||
CallLaterResult,
|
||||
call_later,
|
||||
create_looping_call,
|
||||
)
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.defer import (
|
||||
_defer_sleep_async,
|
||||
_schedule_coro,
|
||||
deferred_from_coro,
|
||||
maybe_deferred_to_future,
|
||||
)
|
||||
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
|
||||
from scrapy.utils.defer import _defer_sleep_async, deferred_from_coro
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
|
||||
from twisted.internet.task import LoopingCall
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Response
|
||||
from scrapy.settings import BaseSettings
|
||||
from scrapy.signalmanager import SignalManager
|
||||
from scrapy.throttler import ThrottlerProtocol, ThrottlerScopeManagerProtocol
|
||||
from scrapy.utils.asyncio import CallLaterResult
|
||||
|
||||
|
||||
@dataclass(slots=True, eq=False)
|
||||
class Slot:
|
||||
class _Slot:
|
||||
"""Downloader slot"""
|
||||
|
||||
concurrency: int
|
||||
|
|
@ -80,46 +71,151 @@ class Slot:
|
|||
)
|
||||
|
||||
|
||||
def _get_concurrency_delay(
|
||||
concurrency: int, spider: Spider, settings: BaseSettings
|
||||
) -> tuple[int, float]:
|
||||
delay: float = settings.getfloat("DOWNLOAD_DELAY")
|
||||
if hasattr(spider, "download_delay"):
|
||||
delay = spider.download_delay
|
||||
Slot = create_deprecated_class(
|
||||
"Slot",
|
||||
_Slot,
|
||||
old_class_path="scrapy.core.downloader.Slot",
|
||||
subclass_warn_message=("{cls} inherits from the deprecated Slot class."),
|
||||
instance_warn_message=("The Slot class is deprecated."),
|
||||
)
|
||||
|
||||
if hasattr(spider, "max_concurrent_requests"): # pragma: no cover
|
||||
warn_on_deprecated_spider_attribute(
|
||||
"max_concurrent_requests", "CONCURRENT_REQUESTS"
|
||||
|
||||
class _DeprecatedSlotView:
|
||||
"""Deprecated per-domain slot view backed by the downloader and throttler."""
|
||||
|
||||
__slots__ = ("_downloader", "_key", "_scope")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
downloader: Downloader,
|
||||
key: str,
|
||||
scope: ThrottlerScopeManagerProtocol,
|
||||
) -> None:
|
||||
self._downloader = downloader
|
||||
self._key = key
|
||||
self._scope = scope
|
||||
|
||||
@property
|
||||
def active(self) -> set[Request]:
|
||||
return {
|
||||
r
|
||||
for r in self._downloader.active
|
||||
if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key
|
||||
}
|
||||
|
||||
@property
|
||||
def transferring(self) -> set[Request]:
|
||||
return {
|
||||
r
|
||||
for r in self._downloader._transferring
|
||||
if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key
|
||||
}
|
||||
|
||||
# This deprecated view reads throttler scope state from private attributes
|
||||
# of the default scope manager rather than through the scope manager
|
||||
# protocol: these are read-only compatibility accessors, so keeping them off
|
||||
# the protocol avoids forcing custom THROTTLER_SCOPE_MANAGER implementations
|
||||
# to provide members that only exist to feed this shim. A custom manager that
|
||||
# lacks the attribute simply falls back to the historical default.
|
||||
@property
|
||||
def lastseen(self) -> float:
|
||||
return getattr(self._scope, "_last_seen", None) or 0.0
|
||||
|
||||
@property
|
||||
def delay(self) -> float:
|
||||
return getattr(self._scope, "_delay", 0.0)
|
||||
|
||||
@delay.setter
|
||||
def delay(self, value: float) -> None:
|
||||
self._scope.set_base_delay(value, only_increase=False)
|
||||
|
||||
@property
|
||||
def randomize_delay(self) -> bool:
|
||||
return bool(getattr(self._scope, "_jitter", None))
|
||||
|
||||
@property
|
||||
def concurrency(self) -> int:
|
||||
warnings.warn(
|
||||
"Slot.concurrency is deprecated. Per-slot concurrency limits are "
|
||||
"now managed by the throttling system.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
concurrency = spider.max_concurrent_requests
|
||||
return getattr(self._scope, "_concurrency", None) or 0
|
||||
|
||||
return concurrency, delay
|
||||
def free_transfer_slots(self) -> int:
|
||||
concurrency = getattr(self._scope, "_concurrency", None) or 0
|
||||
return concurrency - len(self.transferring)
|
||||
|
||||
def download_delay(self) -> float:
|
||||
delay = self.delay
|
||||
if self.randomize_delay:
|
||||
return random.uniform(0.5 * delay, 1.5 * delay) # noqa: S311
|
||||
return delay
|
||||
|
||||
def close(self) -> None:
|
||||
pass
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"_DeprecatedSlotView({self._key!r})"
|
||||
|
||||
|
||||
class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]):
|
||||
"""Deprecated mapping view of active downloads, keyed by slot name."""
|
||||
|
||||
__slots__ = ("_downloader", "_throttler")
|
||||
|
||||
def __init__(self, downloader: Downloader, throttler: ThrottlerProtocol) -> None:
|
||||
self._downloader = downloader
|
||||
self._throttler = throttler
|
||||
|
||||
def _active_keys(self) -> set[str]:
|
||||
return {
|
||||
r.meta[Downloader.DOWNLOAD_SLOT]
|
||||
for r in self._downloader.active
|
||||
if Downloader.DOWNLOAD_SLOT in r.meta
|
||||
}
|
||||
|
||||
def __getitem__(self, key: str) -> _DeprecatedSlotView:
|
||||
if key not in self._active_keys():
|
||||
raise KeyError(key)
|
||||
scope = self._throttler.get_scope_manager(key)
|
||||
return _DeprecatedSlotView(self._downloader, key, scope)
|
||||
|
||||
def __iter__(self) -> Iterator[str]:
|
||||
return iter(self._active_keys())
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self._active_keys())
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
return key in self._active_keys()
|
||||
|
||||
|
||||
class Downloader:
|
||||
DOWNLOAD_SLOT = "download_slot"
|
||||
_SLOT_GC_INTERVAL: float = 60.0 # seconds
|
||||
|
||||
def __init__(self, crawler: Crawler):
|
||||
self.crawler: Crawler = crawler
|
||||
self.settings: BaseSettings = crawler.settings
|
||||
self.signals: SignalManager = crawler.signals
|
||||
self.slots: dict[str, Slot] = {}
|
||||
self.active: set[Request] = set()
|
||||
self._transferring: set[Request] = set()
|
||||
self.handlers: DownloadHandlers = DownloadHandlers(crawler)
|
||||
self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS")
|
||||
self.domain_concurrency: int = self.settings.getint(
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN"
|
||||
)
|
||||
self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
|
||||
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
|
||||
self.middleware: DownloaderMiddlewareManager = (
|
||||
DownloaderMiddlewareManager.from_crawler(crawler)
|
||||
)
|
||||
self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None
|
||||
self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(
|
||||
"DOWNLOAD_SLOTS"
|
||||
)
|
||||
if self.per_slot_settings:
|
||||
warnings.warn(
|
||||
"The DOWNLOAD_SLOTS setting is deprecated. Use THROTTLER_SCOPES for "
|
||||
"per-domain configuration instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
@inlineCallbacks
|
||||
@_warn_spider_arg
|
||||
|
|
@ -140,94 +236,63 @@ class Downloader:
|
|||
def needs_backout(self) -> bool:
|
||||
return len(self.active) >= self.total_concurrency
|
||||
|
||||
@_warn_spider_arg
|
||||
def _get_slot(
|
||||
self, request: Request, spider: Spider | None = None
|
||||
) -> tuple[str, Slot]:
|
||||
key = self.get_slot_key(request)
|
||||
if key not in self.slots:
|
||||
assert self.crawler.spider
|
||||
slot_settings = self.per_slot_settings.get(key, {})
|
||||
conc = self.ip_concurrency or self.domain_concurrency
|
||||
conc, delay = _get_concurrency_delay(
|
||||
conc, self.crawler.spider, self.settings
|
||||
)
|
||||
conc, delay = (
|
||||
slot_settings.get("concurrency", conc),
|
||||
slot_settings.get("delay", delay),
|
||||
)
|
||||
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
|
||||
new_slot = Slot(conc, delay, randomize_delay)
|
||||
self.slots[key] = new_slot
|
||||
self._start_slot_gc()
|
||||
@property
|
||||
def domain_concurrency(self) -> int:
|
||||
warnings.warn(
|
||||
"Downloader.domain_concurrency is deprecated. Per-domain concurrency "
|
||||
"limits are now managed by the throttling system.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
|
||||
|
||||
return key, self.slots[key]
|
||||
@property
|
||||
def randomize_delay(self) -> bool:
|
||||
warnings.warn(
|
||||
"Downloader.randomize_delay is deprecated. Delay randomization is now "
|
||||
"managed by the throttling system.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
|
||||
|
||||
@property
|
||||
def slots(self) -> _DeprecatedSlotsView:
|
||||
warnings.warn(
|
||||
"Downloader.slots is deprecated. Use the throttler API instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
assert self.crawler.throttler is not None
|
||||
return _DeprecatedSlotsView(self, self.crawler.throttler)
|
||||
|
||||
def _get_slot_key(self, request: Request) -> str:
|
||||
assert self.crawler.throttler is not None
|
||||
return self.crawler.throttler.get_slot_key(request)
|
||||
|
||||
def get_slot_key(self, request: Request) -> str:
|
||||
# Retained as public backward-compatible API. It mirrors the historical
|
||||
# keying (an explicit download_slot wins, else the domain); the slot key
|
||||
# used at run time comes from the throttler (see _get_slot_key()).
|
||||
meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
|
||||
if meta_slot is not None:
|
||||
return meta_slot
|
||||
return urlparse_cached(request).hostname or ""
|
||||
|
||||
key = urlparse_cached(request).hostname or ""
|
||||
if self.ip_concurrency:
|
||||
key = dnscache.get(key, key)
|
||||
|
||||
return key
|
||||
|
||||
# passed as download_func into self.middleware.download() in self.fetch()
|
||||
async def _enqueue_request(self, request: Request) -> Response:
|
||||
key, slot = self._get_slot(request)
|
||||
key = self._get_slot_key(request)
|
||||
request.meta[self.DOWNLOAD_SLOT] = key
|
||||
slot.active.add(request)
|
||||
self.signals.send_catch_log(
|
||||
signal=signals.request_reached_downloader,
|
||||
request=request,
|
||||
spider=self.crawler.spider,
|
||||
)
|
||||
d: Deferred[Response] = Deferred()
|
||||
slot.queue.append((request, d))
|
||||
self._process_queue(slot)
|
||||
return await self._download(request)
|
||||
|
||||
async def _download(self, request: Request) -> Response:
|
||||
self._transferring.add(request)
|
||||
try:
|
||||
return await maybe_deferred_to_future(d) # fired in _wait_for_download()
|
||||
finally:
|
||||
slot.active.remove(request)
|
||||
|
||||
def _process_queue(self, slot: Slot) -> None:
|
||||
if slot.latercall:
|
||||
# block processing until slot.latercall is called
|
||||
return
|
||||
|
||||
# Delay queue processing if a download_delay is configured
|
||||
now = monotonic()
|
||||
delay = slot.download_delay()
|
||||
if delay:
|
||||
penalty = delay - now + slot.lastseen
|
||||
if penalty > 0:
|
||||
slot.latercall = call_later(penalty, self._latercall, slot)
|
||||
return
|
||||
|
||||
# Process enqueued requests if there are free slots to transfer for this slot
|
||||
while slot.queue and slot.free_transfer_slots() > 0:
|
||||
slot.lastseen = now
|
||||
request, queue_dfd = slot.queue.popleft()
|
||||
_schedule_coro(self._wait_for_download(slot, request, queue_dfd))
|
||||
# prevent burst if inter-request delays were configured
|
||||
if delay:
|
||||
self._process_queue(slot)
|
||||
break
|
||||
|
||||
def _latercall(self, slot: Slot) -> None:
|
||||
slot.latercall = None
|
||||
self._process_queue(slot)
|
||||
|
||||
async def _download(self, slot: Slot, request: Request) -> Response:
|
||||
# The order is very important for the following logic. Do not change!
|
||||
slot.transferring.add(request)
|
||||
try:
|
||||
# 1. Download the response
|
||||
response: Response = await self.handlers.download_request_async(request)
|
||||
# 2. Notify response_downloaded listeners about the recent download
|
||||
# before querying queue for next request
|
||||
self.signals.send_catch_log(
|
||||
signal=signals.response_downloaded,
|
||||
response=response,
|
||||
|
|
@ -239,46 +304,12 @@ class Downloader:
|
|||
await _defer_sleep_async()
|
||||
raise
|
||||
finally:
|
||||
# 3. After response arrives, remove the request from transferring
|
||||
# state to free up the transferring slot so it can be used by the
|
||||
# following requests (perhaps those which came from the downloader
|
||||
# middleware itself)
|
||||
slot.transferring.remove(request)
|
||||
self._process_queue(slot)
|
||||
self._transferring.discard(request)
|
||||
self.signals.send_catch_log(
|
||||
signal=signals.request_left_downloader,
|
||||
request=request,
|
||||
spider=self.crawler.spider,
|
||||
)
|
||||
|
||||
async def _wait_for_download(
|
||||
self, slot: Slot, request: Request, queue_dfd: Deferred[Response]
|
||||
) -> None:
|
||||
try:
|
||||
response = await self._download(slot, request)
|
||||
except Exception:
|
||||
queue_dfd.errback(Failure())
|
||||
else:
|
||||
queue_dfd.callback(response) # awaited in _enqueue_request()
|
||||
|
||||
def close(self) -> None:
|
||||
self._stop_slot_gc()
|
||||
for slot in self.slots.values():
|
||||
slot.close()
|
||||
|
||||
def _slot_gc(self, age: float = 60) -> None:
|
||||
mintime = monotonic() - age
|
||||
for key, slot in list(self.slots.items()):
|
||||
if not slot.active and slot.lastseen + slot.delay < mintime:
|
||||
self.slots.pop(key).close()
|
||||
|
||||
def _start_slot_gc(self) -> None:
|
||||
if self._slot_gc_loop:
|
||||
return
|
||||
self._slot_gc_loop = create_looping_call(self._slot_gc)
|
||||
self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False)
|
||||
|
||||
def _stop_slot_gc(self) -> None:
|
||||
if self._slot_gc_loop:
|
||||
self._slot_gc_loop.stop()
|
||||
self._slot_gc_loop = None
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -7,11 +7,34 @@ from .base import BaseDownloadHandler
|
|||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import BaseSettings
|
||||
|
||||
|
||||
class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):
|
||||
"""Base class for built-in HTTP download handlers."""
|
||||
|
||||
@staticmethod
|
||||
def _max_per_host_concurrency(settings: BaseSettings) -> int:
|
||||
"""Highest per-host concurrency the throttler may admit: the per-domain
|
||||
limit, the default ``other``-scope limit, and any explicit
|
||||
:setting:`THROTTLER_SCOPES` concurrency.
|
||||
|
||||
Since :setting:`CONCURRENT_REQUESTS` caps the total number of requests in
|
||||
flight, no host can ever exceed it, so it is also the upper bound of the
|
||||
result.
|
||||
"""
|
||||
global_concurrency = settings.getint("CONCURRENT_REQUESTS")
|
||||
candidates = [
|
||||
settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"),
|
||||
settings.getint("THROTTLER_SCOPE_CONCURRENCY"),
|
||||
]
|
||||
candidates += [
|
||||
int(scope["concurrency"])
|
||||
for scope in settings.getdict("THROTTLER_SCOPES").values()
|
||||
if "concurrency" in scope
|
||||
]
|
||||
return min(max(candidates), global_concurrency)
|
||||
|
||||
def __init__(self, crawler: Crawler):
|
||||
super().__init__(crawler)
|
||||
self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")
|
||||
|
|
|
|||
|
|
@ -84,11 +84,8 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
|
|||
crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
|
||||
)
|
||||
self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
|
||||
# these are useful for many handlers but used in different ways by them
|
||||
self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS")
|
||||
self._pool_size_per_host: int = crawler.settings.getint(
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN"
|
||||
)
|
||||
self._pool_size_per_host: int = self._max_per_host_concurrency(crawler.settings)
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
|
|
|
|||
|
|
@ -92,8 +92,8 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
|
|||
from twisted.internet import reactor
|
||||
|
||||
self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
|
||||
self._pool.maxPersistentPerHost = crawler.settings.getint(
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN"
|
||||
self._pool.maxPersistentPerHost = self._max_per_host_concurrency(
|
||||
crawler.settings
|
||||
)
|
||||
self._pool._factory.noisy = False
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from scrapy.exceptions import (
|
|||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.asyncio import (
|
||||
AsyncioLoopingCall,
|
||||
call_later,
|
||||
create_looping_call,
|
||||
is_asyncio_available,
|
||||
)
|
||||
|
|
@ -57,6 +58,7 @@ if TYPE_CHECKING:
|
|||
from scrapy.settings import BaseSettings, Settings
|
||||
from scrapy.signalmanager import SignalManager
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.asyncio import CallLaterResult
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -129,6 +131,25 @@ class ExecutionEngine:
|
|||
self._start_request_processing_awaitable: (
|
||||
asyncio.Future[None] | Deferred[None] | None
|
||||
) = None
|
||||
# Requests currently held by the throttler, waiting for their
|
||||
# scopes to allow them through to the downloader.
|
||||
self._throttler_waiting: set[Request] = set()
|
||||
# Number of in-flight asynchronous enqueue operations (see
|
||||
# ``_enqueue_request_async``), so the spider is not considered idle
|
||||
# while a request is still on its way into the scheduler.
|
||||
self._scheduling: int = 0
|
||||
# A coalesced wakeup timer, armed when the scheduler reports (through
|
||||
# ``get_next_request_delay``) that every pending request is time-blocked.
|
||||
self._delay_wakeup: CallLaterResult | None = None
|
||||
# The scheduler's ``get_next_request_delay`` bound method, cached once
|
||||
# per crawl in ``open_spider_async`` (``None`` if the scheduler does not
|
||||
# expose one). Used both to arm the wakeup timer and to tell whether the
|
||||
# scheduler holds time-blocked requests itself.
|
||||
self._get_next_request_delay: Callable[[], float | None] | None = None
|
||||
# Whether the scheduler exposes enqueue_request_async, cached once per
|
||||
# crawl in ``open_spider_async``; checked for every scheduled request.
|
||||
self._scheduler_enqueues_async: bool = False
|
||||
self._throttler_backout_warned: bool = False
|
||||
downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"])
|
||||
try:
|
||||
self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class(
|
||||
|
|
@ -262,9 +283,16 @@ class ExecutionEngine:
|
|||
|
||||
def pause(self) -> None:
|
||||
self.paused = True
|
||||
self._cancel_delay_wakeup()
|
||||
|
||||
def unpause(self) -> None:
|
||||
self.paused = False
|
||||
# pause() cancels the coalesced delay wakeup and the loop stops
|
||||
# re-running itself while paused, so re-run it here: this re-arms the
|
||||
# wakeup and keeps a crawl held only by a time-based gate (nothing in
|
||||
# flight to re-run the loop from _download) from stalling.
|
||||
if self._slot is not None:
|
||||
self._slot.nextcall.schedule()
|
||||
|
||||
async def _process_start_next(self) -> None:
|
||||
"""Processes the next item or request from Spider.start().
|
||||
|
|
@ -331,13 +359,46 @@ class ExecutionEngine:
|
|||
if self._slot is None or self._slot.closing is not None or self.paused:
|
||||
return
|
||||
|
||||
self._cancel_delay_wakeup()
|
||||
|
||||
while not self.needs_backout():
|
||||
if not self._start_scheduled_request():
|
||||
break
|
||||
|
||||
self._maybe_arm_delay_wakeup()
|
||||
|
||||
if self.spider_is_idle() and self._slot.close_if_idle:
|
||||
self._spider_idle()
|
||||
|
||||
def _cancel_delay_wakeup(self) -> None:
|
||||
if self._delay_wakeup is not None:
|
||||
self._delay_wakeup.cancel()
|
||||
self._delay_wakeup = None
|
||||
|
||||
def _maybe_arm_delay_wakeup(self) -> None:
|
||||
"""Arm a single coalesced wakeup when the scheduler reports that every
|
||||
pending request is time-blocked.
|
||||
|
||||
Concurrency-blocked states do not need this: a freed slot already
|
||||
re-runs the loop from :meth:`_download`'s ``finally``. Only time-based
|
||||
gates (delay, backoff, quota windows) need a timer, since nothing else
|
||||
would re-run the loop while they are closed.
|
||||
"""
|
||||
assert self._slot is not None
|
||||
if (
|
||||
self._get_next_request_delay is None
|
||||
or not self._slot.scheduler.has_pending_requests()
|
||||
):
|
||||
return
|
||||
# A positive delay means a time-based gate nothing else would wake us
|
||||
# for. ``None`` (nothing time-blocked) and ``0`` (something is ready but
|
||||
# could not be sent, e.g. the downloader is full) are handled elsewhere:
|
||||
# a freed slot re-runs the loop, and a ``0``-second timer would busy-loop.
|
||||
delay = self._get_next_request_delay()
|
||||
if delay is None or delay <= 0:
|
||||
return
|
||||
self._delay_wakeup = call_later(delay, self._slot.nextcall.schedule)
|
||||
|
||||
def needs_backout(self) -> bool:
|
||||
"""Returns ``True`` if no more requests can be sent at the moment, or
|
||||
``False`` otherwise.
|
||||
|
|
@ -351,6 +412,42 @@ class ExecutionEngine:
|
|||
or bool(self._slot.closing)
|
||||
or self.downloader.needs_backout()
|
||||
or self.scraper.slot.needs_backout()
|
||||
or self._throttler_needs_backout()
|
||||
)
|
||||
|
||||
def _throttler_needs_backout(self) -> bool:
|
||||
"""Apply backpressure while requests are held by the throttler.
|
||||
|
||||
Requests waiting on throttling do not occupy a downloader concurrency
|
||||
slot, so without this check the engine would keep draining the
|
||||
scheduler into pending throttling waits. Count them together with the
|
||||
in-flight requests against the global concurrency limit.
|
||||
"""
|
||||
if not self._throttler_waiting:
|
||||
return False
|
||||
if (
|
||||
len(self._throttler_waiting) + len(self.downloader.active)
|
||||
< self.downloader.total_concurrency
|
||||
):
|
||||
return False
|
||||
self._maybe_warn_throttler_backout()
|
||||
return True
|
||||
|
||||
def _maybe_warn_throttler_backout(self) -> None:
|
||||
if self._throttler_backout_warned:
|
||||
return
|
||||
# A throttler-aware scheduler holds time-blocked requests itself instead
|
||||
# of letting them pile up in _throttler_waiting and back-pressure the
|
||||
# engine here, so the warning does not apply when one is in use.
|
||||
if self._get_next_request_delay is not None:
|
||||
return
|
||||
self._throttler_backout_warned = True
|
||||
logger.warning(
|
||||
"Throttling is holding requests back and they are now consuming the "
|
||||
"global concurrency budget while they wait for a free slot. Consider "
|
||||
"switching to scrapy.core.scheduler.ThrottlerAwareScheduler, which "
|
||||
"holds throttled requests without consuming concurrency slots.",
|
||||
extra={"spider": self.spider},
|
||||
)
|
||||
|
||||
def _remove_request(self, _: Any, request: Request) -> None:
|
||||
|
|
@ -426,6 +523,10 @@ class ExecutionEngine:
|
|||
return False
|
||||
if self.downloader.active: # downloader has pending requests
|
||||
return False
|
||||
if self._throttler_waiting: # requests held by the throttler
|
||||
return False
|
||||
if self._scheduling: # requests still on their way into the scheduler
|
||||
return False
|
||||
if self._start is not None: # not all start requests are handled
|
||||
return False
|
||||
return not self._slot.scheduler.has_pending_requests()
|
||||
|
|
@ -438,6 +539,7 @@ class ExecutionEngine:
|
|||
self._slot.nextcall.schedule() # type: ignore[union-attr]
|
||||
|
||||
def _schedule_request(self, request: Request) -> None:
|
||||
assert self._slot is not None
|
||||
request_scheduled_result = self.signals.send_catch_log(
|
||||
signals.request_scheduled,
|
||||
request=request,
|
||||
|
|
@ -447,11 +549,39 @@ class ExecutionEngine:
|
|||
for _, result in request_scheduled_result:
|
||||
if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest):
|
||||
return
|
||||
if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
|
||||
scheduler = self._slot.scheduler
|
||||
if self._scheduler_enqueues_async:
|
||||
self._scheduling += 1
|
||||
_schedule_coro(self._enqueue_request_async(request))
|
||||
return
|
||||
if not scheduler.enqueue_request(request):
|
||||
self.signals.send_catch_log(
|
||||
signals.request_dropped, request=request, spider=self.spider
|
||||
)
|
||||
|
||||
async def _enqueue_request_async(self, request: Request) -> None:
|
||||
# _scheduling is incremented in _schedule_request before this coroutine
|
||||
# is scheduled, so it must be decremented on every path (hence finally),
|
||||
# otherwise spider_is_idle() never reports idle.
|
||||
try:
|
||||
if self._slot is None:
|
||||
return
|
||||
stored = await self._slot.scheduler.enqueue_request_async(request) # type: ignore[attr-defined]
|
||||
if not stored:
|
||||
self.signals.send_catch_log(
|
||||
signals.request_dropped, request=request, spider=self.spider
|
||||
)
|
||||
except Exception:
|
||||
logger.error(
|
||||
"Error while enqueuing request",
|
||||
exc_info=True,
|
||||
extra={"spider": self.spider},
|
||||
)
|
||||
finally:
|
||||
self._scheduling -= 1
|
||||
if self._slot is not None:
|
||||
self._slot.nextcall.schedule()
|
||||
|
||||
def download(self, request: Request) -> Deferred[Response]:
|
||||
"""Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
|
||||
warnings.warn(
|
||||
|
|
@ -481,6 +611,20 @@ class ExecutionEngine:
|
|||
return await self.download_async(response_or_request)
|
||||
return response_or_request
|
||||
|
||||
@inlineCallbacks
|
||||
def _acquire_throttler(
|
||||
self, request: Request
|
||||
) -> Generator[Deferred[Any], Any, None]:
|
||||
"""Wait at the throttling gate before *request* is sent, tracking it as
|
||||
held meanwhile."""
|
||||
self._throttler_waiting.add(request)
|
||||
throttler = self.crawler.throttler
|
||||
assert throttler is not None
|
||||
try:
|
||||
yield deferred_from_coro(throttler.acquire(request))
|
||||
finally:
|
||||
self._throttler_waiting.discard(request)
|
||||
|
||||
@inlineCallbacks
|
||||
def _download(
|
||||
self, request: Request
|
||||
|
|
@ -488,8 +632,14 @@ class ExecutionEngine:
|
|||
assert self._slot is not None # typing
|
||||
assert self.spider is not None
|
||||
|
||||
self._slot.add_request(request)
|
||||
throttler = self.crawler.throttler
|
||||
assert throttler is not None
|
||||
# A throttler-aware scheduler reserves the request (a concurrency slot)
|
||||
# before handing it here, so everything past this point runs inside the
|
||||
# try/finally that releases it, even if add_request() were to raise.
|
||||
try:
|
||||
self._slot.add_request(request)
|
||||
yield self._acquire_throttler(request)
|
||||
result: Response | Request
|
||||
if self._downloader_fetch_needs_spider:
|
||||
result = yield self.downloader.fetch(request, self.spider)
|
||||
|
|
@ -515,6 +665,7 @@ class ExecutionEngine:
|
|||
)
|
||||
return result
|
||||
finally:
|
||||
throttler.release(request)
|
||||
self._slot.nextcall.schedule()
|
||||
|
||||
def open_spider(
|
||||
|
|
@ -537,6 +688,10 @@ class ExecutionEngine:
|
|||
self.spider = self.crawler.spider
|
||||
nextcall = CallLaterOnce(self._start_scheduled_requests)
|
||||
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
|
||||
self._get_next_request_delay = getattr(
|
||||
scheduler, "get_next_request_delay", None
|
||||
)
|
||||
self._scheduler_enqueues_async = hasattr(scheduler, "enqueue_request_async")
|
||||
self._slot = _Slot(close_if_idle, nextcall, scheduler)
|
||||
self._start = await self.scraper.spidermw.process_start()
|
||||
if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
|
||||
|
|
@ -613,6 +768,8 @@ class ExecutionEngine:
|
|||
"Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
|
||||
)
|
||||
|
||||
self._cancel_delay_wakeup()
|
||||
|
||||
try:
|
||||
await self._slot.close()
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, cast
|
|||
from twisted.internet.defer import Deferred # noqa: TC002
|
||||
|
||||
from scrapy.spiders import Spider # noqa: TC001
|
||||
from scrapy.throttler import iter_scopes
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
|
||||
|
|
@ -25,6 +26,7 @@ if TYPE_CHECKING:
|
|||
from scrapy.http.request import Request
|
||||
from scrapy.pqueues import ScrapyPriorityQueue
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.throttler import ThrottlerProtocol
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -63,6 +65,33 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
|
|||
plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`.
|
||||
|
||||
The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with.
|
||||
|
||||
Asynchronous API
|
||||
================
|
||||
|
||||
Beyond the minimal interface above, the engine also uses the following
|
||||
optional members when a scheduler defines them, to support asynchronous
|
||||
scheduling. :meth:`next_request` itself stays synchronous.
|
||||
|
||||
.. method:: enqueue_request_async(request)
|
||||
:async:
|
||||
|
||||
Asynchronous counterpart of :meth:`enqueue_request`, following the same
|
||||
return contract. When a scheduler defines it, the engine awaits it
|
||||
*instead of* calling :meth:`enqueue_request`. Define it when enqueuing a
|
||||
request needs to ``await`` (e.g. to resolve throttler scopes
|
||||
asynchronously).
|
||||
|
||||
.. method:: get_next_request_delay()
|
||||
|
||||
Return the number of seconds until some pending request that
|
||||
:meth:`next_request` is currently withholding for a time-based reason
|
||||
becomes available, or ``None`` if no pending request is time-blocked.
|
||||
When a scheduler defines it, the engine uses it to schedule a single
|
||||
wakeup after :meth:`next_request` returns ``None`` while requests remain
|
||||
pending, so a crawl held back only by time does not stall.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
|
|
@ -209,9 +238,8 @@ class Scheduler(BaseScheduler):
|
|||
-------------------------
|
||||
|
||||
While pending requests are below the configured values of
|
||||
:setting:`CONCURRENT_REQUESTS` or
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
|
||||
concurrently.
|
||||
:setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLER_SCOPE_CONCURRENCY`,
|
||||
those requests are sent concurrently.
|
||||
|
||||
As a result, the first few requests of a crawl may not follow the desired
|
||||
order. Lowering those settings to ``1`` enforces the desired order except
|
||||
|
|
@ -374,12 +402,21 @@ class Scheduler(BaseScheduler):
|
|||
if not request.dont_filter and self.df.request_seen(request):
|
||||
self.df.log(request, self.spider)
|
||||
return False
|
||||
dqok = self._dqpush(request)
|
||||
return self._store(request)
|
||||
|
||||
def _store(self, request: Request, *push_args: Any) -> bool:
|
||||
"""Push *request* into the disk queue, falling back to the memory queue,
|
||||
and increment the relevant ``scheduler/enqueued*`` stats.
|
||||
|
||||
Extra positional arguments are forwarded to the underlying queue
|
||||
``push`` calls; subclasses that store requests under additional keys
|
||||
(e.g. a throttler scope set) pass them through here.
|
||||
"""
|
||||
assert self.stats is not None
|
||||
if dqok:
|
||||
if self._dqpush(request, *push_args):
|
||||
self.stats.inc_value("scheduler/enqueued/disk")
|
||||
else:
|
||||
self._mqpush(request)
|
||||
self._mqpush(request, *push_args)
|
||||
self.stats.inc_value("scheduler/enqueued/memory")
|
||||
self.stats.inc_value("scheduler/enqueued")
|
||||
return True
|
||||
|
|
@ -411,11 +448,11 @@ class Scheduler(BaseScheduler):
|
|||
"""
|
||||
return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs)
|
||||
|
||||
def _dqpush(self, request: Request) -> bool:
|
||||
def _dqpush(self, request: Request, *push_args: Any) -> bool:
|
||||
if self.dqs is None:
|
||||
return False
|
||||
try:
|
||||
self.dqs.push(request)
|
||||
self.dqs.push(request, *push_args)
|
||||
except ValueError as e: # non serializable request
|
||||
if self.logunser:
|
||||
msg = (
|
||||
|
|
@ -435,8 +472,8 @@ class Scheduler(BaseScheduler):
|
|||
return False
|
||||
return True
|
||||
|
||||
def _mqpush(self, request: Request) -> None:
|
||||
self.mqs.push(request)
|
||||
def _mqpush(self, request: Request, *push_args: Any) -> None:
|
||||
self.mqs.push(request, *push_args)
|
||||
|
||||
def _dqpop(self) -> Request | None:
|
||||
if self.dqs is not None:
|
||||
|
|
@ -496,3 +533,69 @@ class Scheduler(BaseScheduler):
|
|||
def _write_dqs_state(self, dqdir: str, state: Any) -> None:
|
||||
with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
|
||||
json.dump(state, f)
|
||||
|
||||
|
||||
class ThrottlerAwareScheduler(Scheduler):
|
||||
"""A :setting:`SCHEDULER` that only ever hands the engine requests that
|
||||
their :ref:`throttler scopes <throttler-scopes>` allow to be sent **right
|
||||
now**.
|
||||
|
||||
The default scheduler hands requests to the engine as concurrency allows
|
||||
and lets them wait at the throttling gate
|
||||
(:meth:`~scrapy.throttler.ThrottlerProtocol.acquire`) while holding
|
||||
a concurrency slot. When a crawl mixes heavily-throttled scopes with
|
||||
unthrottled ones, enough throttled requests waiting on a clock can fill
|
||||
:setting:`CONCURRENT_REQUESTS` and starve unthrottled requests that could be
|
||||
sent right away. This scheduler instead withholds a request until it is
|
||||
sendable, so held-back requests occupy neither concurrency slots nor memory
|
||||
(beyond what is needed to track each distinct scope set) and cannot starve
|
||||
other scopes.
|
||||
|
||||
It also honors the per-request :reqmeta:`delay`, holding an
|
||||
individual request back without blocking others that share its scopes, which
|
||||
the default scheduler cannot do.
|
||||
|
||||
When several requests could be sent at the same time, the one with the
|
||||
highest request :attr:`~scrapy.Request.priority` is sent first; ties are
|
||||
broken by preferring the least-busy scopes.
|
||||
|
||||
It requires :setting:`SCHEDULER_PRIORITY_QUEUE` to be set to
|
||||
:class:`~scrapy.pqueues.ThrottlerAwarePriorityQueue` (or a compatible
|
||||
subclass).
|
||||
"""
|
||||
|
||||
def open(self, spider: Spider) -> Deferred[None] | None:
|
||||
result = super().open(spider)
|
||||
if not hasattr(self.mqs, "get_next_request_delay"):
|
||||
raise ValueError(
|
||||
f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be "
|
||||
f"set to a throttler-aware priority queue such as "
|
||||
f"scrapy.pqueues.ThrottlerAwarePriorityQueue, but the "
|
||||
f"configured one ({type(self.mqs).__name__}) is not."
|
||||
)
|
||||
assert self.crawler is not None
|
||||
assert self.crawler.throttler is not None
|
||||
self._throttler: ThrottlerProtocol = self.crawler.throttler
|
||||
return result
|
||||
|
||||
def enqueue_request(self, request: Request) -> bool:
|
||||
raise RuntimeError(
|
||||
"ThrottlerAwareScheduler requires the asynchronous enqueue path; "
|
||||
"enqueue_request_async() is used by the engine instead of "
|
||||
"enqueue_request()."
|
||||
)
|
||||
|
||||
async def enqueue_request_async(self, request: Request) -> bool:
|
||||
if not request.dont_filter and self.df.request_seen(request):
|
||||
self.df.log(request, self.spider)
|
||||
return False
|
||||
scope_set = frozenset(iter_scopes(await self._throttler.get_scopes(request)))
|
||||
return self._store(request, scope_set)
|
||||
|
||||
def get_next_request_delay(self) -> float | None:
|
||||
delays = [
|
||||
delay
|
||||
for pq in (self.mqs, self.dqs)
|
||||
if pq is not None and (delay := pq.get_next_request_delay()) is not None # type: ignore[attr-defined]
|
||||
]
|
||||
return min(delays) if delays else None
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ if TYPE_CHECKING:
|
|||
|
||||
from scrapy.logformatter import LogFormatter
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.throttler import ThrottlerProtocol
|
||||
from scrapy.utils.request import RequestFingerprinterProtocol
|
||||
|
||||
|
||||
|
|
@ -83,6 +84,13 @@ class Crawler:
|
|||
self.stats: StatsCollector | None = None
|
||||
self.logformatter: LogFormatter | None = None
|
||||
self.request_fingerprinter: RequestFingerprinterProtocol | None = None
|
||||
self.throttler: ThrottlerProtocol | None = None
|
||||
"""The throttler of this crawler, an instance of
|
||||
:setting:`THROTTLER`.
|
||||
|
||||
It is ``None`` until the crawl starts. Components can use it to inspect
|
||||
or drive :ref:`throttling <throttling>` at run time, e.g. through
|
||||
:meth:`~scrapy.throttler.ThrottlerProtocol.delay_scope`."""
|
||||
self.spider: Spider | None = None
|
||||
self.engine: ExecutionEngine | None = None
|
||||
|
||||
|
|
@ -96,6 +104,10 @@ class Crawler:
|
|||
return
|
||||
|
||||
self.addons.load_settings(self.settings)
|
||||
self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")
|
||||
self._apply_deprecated_spider_attr(
|
||||
"max_concurrent_requests", "THROTTLER_SCOPE_CONCURRENCY"
|
||||
)
|
||||
self.stats = load_object(self.settings["STATS_CLASS"])(self)
|
||||
|
||||
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
|
||||
|
|
@ -105,6 +117,10 @@ class Crawler:
|
|||
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
|
||||
self,
|
||||
)
|
||||
self.throttler = build_from_crawler(
|
||||
load_object(self.settings["THROTTLER"]),
|
||||
self,
|
||||
)
|
||||
|
||||
use_reactor = self.settings.getbool("TWISTED_REACTOR_ENABLED")
|
||||
if use_reactor:
|
||||
|
|
@ -151,6 +167,30 @@ class Crawler:
|
|||
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
|
||||
)
|
||||
|
||||
def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None:
|
||||
"""Bridge a deprecated spider attribute onto *setting*, warning about
|
||||
the deprecation (and about being ignored when *setting* is already set
|
||||
at spider or higher priority)."""
|
||||
spider = self.spider if self.spider is not None else self.spidercls
|
||||
if not hasattr(spider, attr):
|
||||
return
|
||||
if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]:
|
||||
warnings.warn(
|
||||
f"The {attr!r} spider attribute is deprecated. It is also being "
|
||||
f"ignored because {setting} is already set at spider or higher "
|
||||
f"priority. Remove the {attr!r} attribute from your spider.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
return
|
||||
warnings.warn(
|
||||
f"The {attr!r} spider attribute is deprecated. Use the {setting} "
|
||||
f"setting or THROTTLER_SCOPES instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=3,
|
||||
)
|
||||
self.settings.set(setting, getattr(spider, attr), priority="spider")
|
||||
|
||||
def _apply_reactorless_default_settings(self) -> None:
|
||||
"""Change some setting defaults when not using a Twisted reactor.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.throttler import iter_scopes
|
||||
from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after
|
||||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.misc import _load_objects
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.throttler import ThrottlerProtocol
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BackoffMiddleware:
|
||||
"""Downloader middleware that drives :ref:`backoff <backoff>` from download
|
||||
outcomes.
|
||||
|
||||
It observes every response and download exception and, for those matching
|
||||
:setting:`BACKOFF_HTTP_CODES` or :setting:`BACKOFF_EXCEPTIONS` (globally or
|
||||
per :setting:`THROTTLER_SCOPES` scope), tells the :ref:`throttler
|
||||
<throttling>` to back off the request's scopes through its
|
||||
:meth:`~scrapy.throttler.ThrottlerProtocol.back_off` API.
|
||||
|
||||
It is enabled by default; set :setting:`BACKOFF_ENABLED` to ``False`` to
|
||||
disable it without removing it from :setting:`DOWNLOADER_MIDDLEWARES`.
|
||||
|
||||
See :ref:`throttling` for details.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler: Crawler):
|
||||
if not crawler.settings.getbool("BACKOFF_ENABLED"):
|
||||
raise NotConfigured
|
||||
assert crawler.throttler is not None
|
||||
self._throttler: ThrottlerProtocol = crawler.throttler
|
||||
settings = crawler.settings
|
||||
self._http_codes: set[int] = {
|
||||
int(code) for code in settings.getlist("BACKOFF_HTTP_CODES")
|
||||
}
|
||||
self._exceptions: tuple[type[BaseException], ...] = _load_objects(
|
||||
settings.getlist("BACKOFF_EXCEPTIONS")
|
||||
)
|
||||
# Per-scope overrides; a scope absent from these uses the globals above.
|
||||
self._scope_http_codes: dict[str, set[int]] = {}
|
||||
self._scope_exceptions: dict[str, tuple[type[BaseException], ...]] = {}
|
||||
# Unions of the globals and every per-scope override, used as a cheap
|
||||
# pre-filter to skip outcomes that no scope backs off on.
|
||||
self._any_http_codes: set[int] = set(self._http_codes)
|
||||
self._any_exceptions: tuple[type[BaseException], ...] = self._exceptions
|
||||
for scope_id, scope_config in settings.getdict("THROTTLER_SCOPES").items():
|
||||
backoff = scope_config.get("backoff") or {}
|
||||
if "http_codes" in backoff:
|
||||
codes = {int(code) for code in backoff["http_codes"]}
|
||||
self._scope_http_codes[scope_id] = codes
|
||||
self._any_http_codes |= codes
|
||||
if "exceptions" in backoff:
|
||||
exceptions = _load_objects(backoff["exceptions"])
|
||||
self._scope_exceptions[scope_id] = exceptions
|
||||
self._any_exceptions += exceptions
|
||||
|
||||
@_warn_spider_arg
|
||||
def process_response(
|
||||
self,
|
||||
request: Request,
|
||||
response: Response,
|
||||
spider: scrapy.Spider | None = None,
|
||||
) -> Response:
|
||||
if (
|
||||
response.status not in self._any_http_codes
|
||||
or "cached" in response.flags
|
||||
or request.meta.get("dont_throttle")
|
||||
):
|
||||
return response
|
||||
matched = [
|
||||
scope
|
||||
for scope in iter_scopes(self._throttler.get_resolved_scopes(request))
|
||||
if response.status in self._scope_http_codes.get(scope, self._http_codes)
|
||||
]
|
||||
if matched:
|
||||
self._throttler.back_off(matched, delay=self._response_delay(response))
|
||||
return response
|
||||
|
||||
@_warn_spider_arg
|
||||
def process_exception(
|
||||
self,
|
||||
request: Request,
|
||||
exception: Exception,
|
||||
spider: scrapy.Spider | None = None,
|
||||
) -> None:
|
||||
if request.meta.get("dont_throttle") or not isinstance(
|
||||
exception, self._any_exceptions
|
||||
):
|
||||
return
|
||||
matched = [
|
||||
scope
|
||||
for scope in iter_scopes(self._throttler.get_resolved_scopes(request))
|
||||
if isinstance(
|
||||
exception, self._scope_exceptions.get(scope, self._exceptions)
|
||||
)
|
||||
]
|
||||
if matched:
|
||||
self._throttler.back_off(matched)
|
||||
|
||||
@staticmethod
|
||||
def _response_delay(response: Response) -> float | None:
|
||||
"""Return the hard minimum delay requested by *response* through a
|
||||
``Retry-After`` or ``RateLimit-Reset`` header, or ``None``."""
|
||||
delays = [
|
||||
delay
|
||||
for delay in (
|
||||
_parse_retry_after(response),
|
||||
_parse_ratelimit_reset(response),
|
||||
)
|
||||
if delay is not None
|
||||
]
|
||||
return max(delays) if delays else None
|
||||
|
|
@ -10,6 +10,7 @@ from scrapy import signals
|
|||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import HtmlResponse, Response
|
||||
from scrapy.spidermiddlewares.referer import RefererMiddleware
|
||||
from scrapy.utils._headers import _parse_retry_after
|
||||
from scrapy.utils.decorators import _warn_spider_arg
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
|
@ -36,6 +37,7 @@ class BaseRedirectMiddleware:
|
|||
raise NotConfigured
|
||||
|
||||
self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES")
|
||||
self.max_delay: float = settings.getfloat("REDIRECT_MAX_DELAY")
|
||||
self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST")
|
||||
self._referer_spider_middleware: RefererMiddleware | None = None
|
||||
|
||||
|
|
@ -174,9 +176,30 @@ class BaseRedirectMiddleware:
|
|||
del redirect_request.headers["Authorization"]
|
||||
|
||||
self.handle_referer(redirect_request, response)
|
||||
self._apply_retry_after(redirect_request, response)
|
||||
|
||||
return redirect_request
|
||||
|
||||
def _apply_retry_after(self, redirect_request: Request, response: Response) -> None:
|
||||
"""Delay the redirect when *response* carries a ``Retry-After`` header.
|
||||
|
||||
The delay is capped at :setting:`REDIRECT_MAX_DELAY` and applied through
|
||||
the :reqmeta:`delay` request metadata key, which holds back
|
||||
this request only (without counting as a :ref:`backoff <backoff>`
|
||||
trigger for its scopes). :setting:`REDIRECT_MAX_DELAY` set to ``0``
|
||||
disables it.
|
||||
"""
|
||||
if not self.max_delay:
|
||||
return
|
||||
retry_after = _parse_retry_after(response)
|
||||
if retry_after is None:
|
||||
return
|
||||
redirect_request.meta["delay"] = min(retry_after, self.max_delay)
|
||||
# This is a fresh request that may inherit an already-honored delay from
|
||||
# its source request's meta; clear that state so the new delay applies.
|
||||
redirect_request.meta.pop("_throttler_delayed", None)
|
||||
redirect_request.meta.pop("_throttler_delay_deadline", None)
|
||||
|
||||
def _redirect_request_using_get(
|
||||
self, request: Request, response: Response, redirect_url: str
|
||||
) -> Request:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
|
|||
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.http.request import NO_CALLBACK
|
||||
|
|
@ -98,7 +99,7 @@ class RobotsTxtMiddleware:
|
|||
assert self.crawler.stats
|
||||
try:
|
||||
resp = await self.crawler.engine.download_async(robotsreq)
|
||||
self._parse_robots(resp, netloc)
|
||||
await self._parse_robots(resp, netloc, request)
|
||||
except Exception as e:
|
||||
if not isinstance(e, IgnoreRequest):
|
||||
logger.error(
|
||||
|
|
@ -115,13 +116,20 @@ class RobotsTxtMiddleware:
|
|||
return await maybe_deferred_to_future(parser)
|
||||
return parser
|
||||
|
||||
def _parse_robots(self, response: Response, netloc: str) -> None:
|
||||
async def _parse_robots(
|
||||
self, response: Response, netloc: str, request: Request
|
||||
) -> None:
|
||||
assert self.crawler.stats
|
||||
self.crawler.stats.inc_value("robotstxt/response_count")
|
||||
self.crawler.stats.inc_value(
|
||||
f"robotstxt/response_status_count/{response.status}"
|
||||
)
|
||||
rp = self._parserimpl.from_crawler(self.crawler, response.body)
|
||||
await self.crawler.signals.send_catch_log_async(
|
||||
signal=signals.robots_parsed,
|
||||
robotparser=rp,
|
||||
request=request,
|
||||
)
|
||||
rp_dfd = self._parsers[netloc]
|
||||
assert isinstance(rp_dfd, Deferred)
|
||||
self._parsers[netloc] = rp
|
||||
|
|
|
|||
|
|
@ -2,17 +2,19 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from warnings import warn
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.core.downloader import Slot
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.http import Response
|
||||
from scrapy.throttler import ThrottlerProtocol
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -24,6 +26,15 @@ class AutoThrottle:
|
|||
if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
|
||||
raise NotConfigured
|
||||
|
||||
warn(
|
||||
"You have set the AUTOTHROTTLE_ENABLED setting to True, however "
|
||||
"the AutoThrottle extension is deprecated; use throttler and "
|
||||
"backoff settings instead: "
|
||||
"https://docs.scrapy.org/en/latest/topics/throttling.html",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
|
||||
self.target_concurrency: float = crawler.settings.getfloat(
|
||||
"AUTOTHROTTLE_TARGET_CONCURRENCY"
|
||||
|
|
@ -33,6 +44,7 @@ class AutoThrottle:
|
|||
f"AUTOTHROTTLE_TARGET_CONCURRENCY "
|
||||
f"({self.target_concurrency!r}) must be higher than 0."
|
||||
)
|
||||
self._started_scopes: set[str] = set()
|
||||
crawler.signals.connect(self._spider_opened, signal=signals.spider_opened)
|
||||
crawler.signals.connect(
|
||||
self._response_downloaded, signal=signals.response_downloaded
|
||||
|
|
@ -45,7 +57,9 @@ class AutoThrottle:
|
|||
def _spider_opened(self, spider: Spider) -> None:
|
||||
self.mindelay = self._min_delay(spider)
|
||||
self.maxdelay = self._max_delay(spider)
|
||||
spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined]
|
||||
self._startdelay = max(
|
||||
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
|
||||
)
|
||||
|
||||
def _min_delay(self, spider: Spider) -> float:
|
||||
s = self.crawler.settings
|
||||
|
|
@ -54,55 +68,51 @@ class AutoThrottle:
|
|||
def _max_delay(self, spider: Spider) -> float:
|
||||
return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
|
||||
|
||||
def _start_delay(self, spider: Spider) -> float:
|
||||
return max(
|
||||
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
|
||||
)
|
||||
|
||||
def _response_downloaded(
|
||||
self, response: Response, request: Request, spider: Spider
|
||||
) -> None:
|
||||
key, slot = self._get_slot(request, spider)
|
||||
throttler = self.crawler.throttler
|
||||
assert throttler is not None
|
||||
latency = request.meta.get("download_latency")
|
||||
if (
|
||||
latency is None
|
||||
or slot is None
|
||||
or request.meta.get("autothrottle_dont_adjust_delay", False) is True
|
||||
):
|
||||
return
|
||||
|
||||
olddelay = slot.delay
|
||||
self._adjust_delay(slot, latency, response)
|
||||
scope_id = urlparse_cached(request).hostname or ""
|
||||
olddelay = self._scope_delay(throttler, scope_id)
|
||||
newdelay = self._adjust_delay(olddelay, latency, response)
|
||||
throttler.set_scope_delay(scope_id, newdelay)
|
||||
if self.debug:
|
||||
diff = slot.delay - olddelay
|
||||
size = len(response.body)
|
||||
conc = len(slot.transferring)
|
||||
logger.info(
|
||||
"slot: %(slot)s | conc:%(concurrency)2d | "
|
||||
"slot: %(slot)s | "
|
||||
"delay:%(delay)5d ms (%(delaydiff)+d) | "
|
||||
"latency:%(latency)5d ms | size:%(size)6d bytes",
|
||||
{
|
||||
"slot": key,
|
||||
"concurrency": conc,
|
||||
"delay": slot.delay * 1000,
|
||||
"delaydiff": diff * 1000,
|
||||
"slot": scope_id,
|
||||
"delay": newdelay * 1000,
|
||||
"delaydiff": (newdelay - olddelay) * 1000,
|
||||
"latency": latency * 1000,
|
||||
"size": size,
|
||||
"size": len(response.body),
|
||||
},
|
||||
extra={"spider": spider},
|
||||
)
|
||||
|
||||
def _get_slot(
|
||||
self, request: Request, spider: Spider
|
||||
) -> tuple[str | None, Slot | None]:
|
||||
key: str | None = request.meta.get("download_slot")
|
||||
if key is None:
|
||||
return None, None
|
||||
assert self.crawler.engine
|
||||
return key, self.crawler.engine.downloader.slots.get(key)
|
||||
def _scope_delay(self, throttler: ThrottlerProtocol, scope_id: str) -> float:
|
||||
"""Return the current delay of *scope_id*, applying AUTOTHROTTLE_START_DELAY
|
||||
the first time the scope is seen."""
|
||||
delay = throttler.get_scope_delay(scope_id)
|
||||
if scope_id not in self._started_scopes:
|
||||
self._started_scopes.add(scope_id)
|
||||
delay = max(delay, self._startdelay)
|
||||
return delay
|
||||
|
||||
def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:
|
||||
"""Define delay adjustment policy"""
|
||||
def _adjust_delay(
|
||||
self, olddelay: float, latency: float, response: Response
|
||||
) -> float:
|
||||
"""Return the new delay given the current *olddelay* and the observed
|
||||
*latency*."""
|
||||
|
||||
# If a server needs `latency` seconds to respond then
|
||||
# we should send a request each `latency/N` seconds
|
||||
|
|
@ -110,7 +120,7 @@ class AutoThrottle:
|
|||
target_delay = latency / self.target_concurrency
|
||||
|
||||
# Adjust the delay to make it closer to target_delay
|
||||
new_delay = (slot.delay + target_delay) / 2.0
|
||||
new_delay = (olddelay + target_delay) / 2.0
|
||||
|
||||
# If target delay is bigger than old delay, then use it instead of mean.
|
||||
# It works better with problematic sites.
|
||||
|
|
@ -123,7 +133,7 @@ class AutoThrottle:
|
|||
# than old one, as error pages (and redirections) are usually small and
|
||||
# so tend to reduce latency, thus provoking a positive feedback by
|
||||
# reducing delay instead of increase.
|
||||
if response.status != 200 and new_delay <= slot.delay:
|
||||
return
|
||||
if response.status != 200 and new_delay <= olddelay:
|
||||
return olddelay
|
||||
|
||||
slot.delay = new_delay
|
||||
return new_delay
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import heapq
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from typing import TYPE_CHECKING, Protocol, cast
|
||||
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
|
|
@ -13,8 +16,8 @@ if TYPE_CHECKING:
|
|||
from typing_extensions import Self
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.core.downloader import Downloader
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.throttler import ScopeID, ThrottlerProtocol
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -235,9 +238,9 @@ class ScrapyPriorityQueue:
|
|||
if self.curprio is None:
|
||||
return None
|
||||
try:
|
||||
queue = self._start_queues[self.curprio]
|
||||
except KeyError:
|
||||
queue = self.queues[self.curprio]
|
||||
except KeyError:
|
||||
queue = self._start_queues[self.curprio]
|
||||
# Protocols can't declare optional members
|
||||
return cast("Request", queue.peek()) # type: ignore[attr-defined]
|
||||
|
||||
|
|
@ -261,24 +264,6 @@ class ScrapyPriorityQueue:
|
|||
)
|
||||
|
||||
|
||||
class DownloaderInterface:
|
||||
def __init__(self, crawler: Crawler):
|
||||
assert crawler.engine
|
||||
self.downloader: Downloader = crawler.engine.downloader
|
||||
|
||||
def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]:
|
||||
return [(self._active_downloads(slot), slot) for slot in possible_slots]
|
||||
|
||||
def get_slot_key(self, request: Request) -> str:
|
||||
return self.downloader.get_slot_key(request)
|
||||
|
||||
def _active_downloads(self, slot: str) -> int:
|
||||
"""Return a number of requests in a Downloader for a given slot"""
|
||||
if slot not in self.downloader.slots:
|
||||
return 0
|
||||
return len(self.downloader.slots[slot].active)
|
||||
|
||||
|
||||
class DownloaderAwarePriorityQueue:
|
||||
"""PriorityQueue which takes Downloader activity into account:
|
||||
domains (slots) with the least amount of active downloads are dequeued
|
||||
|
|
@ -332,11 +317,6 @@ class DownloaderAwarePriorityQueue:
|
|||
*,
|
||||
start_queue_cls: type[QueueProtocol] | None = None,
|
||||
):
|
||||
if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
|
||||
raise ValueError(
|
||||
f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
|
||||
)
|
||||
|
||||
if slot_startprios and not isinstance(slot_startprios, dict):
|
||||
raise ValueError(
|
||||
"DownloaderAwarePriorityQueue accepts "
|
||||
|
|
@ -348,7 +328,8 @@ class DownloaderAwarePriorityQueue:
|
|||
"queue class can be resumed."
|
||||
)
|
||||
|
||||
self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler)
|
||||
assert crawler.throttler is not None
|
||||
self._throttler: ThrottlerProtocol = crawler.throttler
|
||||
self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls
|
||||
self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls
|
||||
self.key: str = key
|
||||
|
|
@ -360,19 +341,19 @@ class DownloaderAwarePriorityQueue:
|
|||
for slot, startprios in slot_startprios.items():
|
||||
self.pqueues[slot] = self.pqfactory(slot, startprios)
|
||||
|
||||
def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str:
|
||||
def _next_slot(self, stats: list[tuple[float, str]], *, update_state: bool) -> str:
|
||||
last = self._last_selected_slot
|
||||
min_active: int | None = None
|
||||
min_load: float | None = None
|
||||
best_slot: str | None = None
|
||||
best_slot_after_last: str | None = None
|
||||
for active, slot in stats:
|
||||
if min_active is None or active < min_active:
|
||||
min_active = active
|
||||
for load, slot in stats:
|
||||
if min_load is None or load < min_load:
|
||||
min_load = load
|
||||
best_slot = slot
|
||||
best_slot_after_last = None
|
||||
if last is not None and slot > last:
|
||||
best_slot_after_last = slot
|
||||
elif active == min_active:
|
||||
elif load == min_load:
|
||||
if best_slot is None or slot < best_slot:
|
||||
best_slot = slot
|
||||
if (
|
||||
|
|
@ -398,8 +379,11 @@ class DownloaderAwarePriorityQueue:
|
|||
start_queue_cls=self._start_queue_cls,
|
||||
)
|
||||
|
||||
def _slot_stats(self) -> list[tuple[float, str]]:
|
||||
return [(self._throttler.get_scope_load(slot), slot) for slot in self.pqueues]
|
||||
|
||||
def pop(self) -> Request | None:
|
||||
stats = self._downloader_interface.stats(self.pqueues)
|
||||
stats = self._slot_stats()
|
||||
|
||||
if not stats:
|
||||
return None
|
||||
|
|
@ -412,7 +396,7 @@ class DownloaderAwarePriorityQueue:
|
|||
return request
|
||||
|
||||
def push(self, request: Request) -> None:
|
||||
slot = self._downloader_interface.get_slot_key(request)
|
||||
slot = self._throttler.get_slot_key(request)
|
||||
if slot not in self.pqueues:
|
||||
self.pqueues[slot] = self.pqfactory(slot)
|
||||
queue = self.pqueues[slot]
|
||||
|
|
@ -425,7 +409,7 @@ class DownloaderAwarePriorityQueue:
|
|||
Raises :exc:`NotImplementedError` if the underlying queue class does
|
||||
not implement a ``peek`` method, which is optional for queues.
|
||||
"""
|
||||
stats = self._downloader_interface.stats(self.pqueues)
|
||||
stats = self._slot_stats()
|
||||
if not stats:
|
||||
return None
|
||||
slot = self._next_slot(stats, update_state=False)
|
||||
|
|
@ -442,3 +426,260 @@ class DownloaderAwarePriorityQueue:
|
|||
|
||||
def __contains__(self, slot: str) -> bool:
|
||||
return slot in self.pqueues
|
||||
|
||||
|
||||
def _scope_set_key(scope_set: frozenset[ScopeID]) -> str:
|
||||
"""Return a reversible, JSON-safe string key for *scope_set*.
|
||||
|
||||
Used both as the in-memory dict key encoding for the on-disk state and to
|
||||
derive the per-scope-set subdirectory name. The encoding is
|
||||
order-independent (the scope ids are sorted)."""
|
||||
return json.dumps(sorted(scope_set))
|
||||
|
||||
|
||||
def _scope_set_from_key(key: str) -> frozenset[ScopeID]:
|
||||
return frozenset(json.loads(key))
|
||||
|
||||
|
||||
class ThrottlerAwarePriorityQueue:
|
||||
"""Priority queue that only ever pops a request that can be sent right now
|
||||
based on its :ref:`throttler scope set <throttler-scopes>` and
|
||||
per-request :reqmeta:`delay`.
|
||||
|
||||
The downstream queue class must support ``peek``.
|
||||
|
||||
Disk persistence
|
||||
================
|
||||
|
||||
.. warning:: The files that this class generates on disk are an
|
||||
implementation detail, and may change without a warning in a future
|
||||
version of Scrapy. Do not rely on the following information for
|
||||
anything other than debugging purposes.
|
||||
|
||||
When instantiated with a non-empty *key* argument, *key* is used as a
|
||||
persistence directory, and inside it this class creates a subdirectory per
|
||||
scope set, named from a path-safe, order-independent encoding of its scope
|
||||
ids.
|
||||
|
||||
For example, a request whose scope set is ``{"example.com",
|
||||
"cost:group-1"}`` is stored under a subdirectory derived in two steps:
|
||||
|
||||
#. The scope ids are sorted and JSON-encoded into an order-independent key
|
||||
(so ``{"example.com", "cost:group-1"}`` and ``{"cost:group-1",
|
||||
"example.com"}`` map to the same one)::
|
||||
|
||||
["cost:group-1", "example.com"]
|
||||
|
||||
#. That key is made path-safe: every character outside ``[A-Za-z0-9-._]``
|
||||
becomes ``_`` (here the ``:`` and the JSON quotes, brackets and spaces;
|
||||
the ``-`` and ``.`` are kept), and an MD5 suffix disambiguates keys that
|
||||
collapse to the same path::
|
||||
|
||||
__cost_group-1____example.com__-fc6ba2aff8f421bf981b662d77739902
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(
|
||||
cls,
|
||||
crawler: Crawler,
|
||||
downstream_queue_cls: type[QueueProtocol],
|
||||
key: str,
|
||||
startprios: dict[str, Iterable[int]] | None = None,
|
||||
*,
|
||||
start_queue_cls: type[QueueProtocol] | None = None,
|
||||
) -> Self:
|
||||
return cls(
|
||||
crawler,
|
||||
downstream_queue_cls,
|
||||
key,
|
||||
startprios,
|
||||
start_queue_cls=start_queue_cls,
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
crawler: Crawler,
|
||||
downstream_queue_cls: type[QueueProtocol],
|
||||
key: str,
|
||||
slot_startprios: dict[str, Iterable[int]] | None = None,
|
||||
*,
|
||||
start_queue_cls: type[QueueProtocol] | None = None,
|
||||
):
|
||||
if slot_startprios and not isinstance(slot_startprios, dict):
|
||||
raise ValueError(
|
||||
"ThrottlerAwarePriorityQueue accepts ``slot_startprios`` as a "
|
||||
f"dict; {slot_startprios.__class__!r} instance is passed. Most "
|
||||
"likely, it means the state is created by an incompatible "
|
||||
"priority queue. Only a crawl started with the same priority "
|
||||
"queue class can be resumed."
|
||||
)
|
||||
|
||||
assert crawler.throttler is not None
|
||||
self._throttler: ThrottlerProtocol = crawler.throttler
|
||||
self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls
|
||||
self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls
|
||||
self.key: str = key
|
||||
self.crawler: Crawler = crawler
|
||||
|
||||
self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {}
|
||||
if slot_startprios:
|
||||
for set_key, startprios in slot_startprios.items():
|
||||
scope_set = _scope_set_from_key(set_key)
|
||||
self.pqueues[scope_set] = self._pqfactory(scope_set, startprios)
|
||||
|
||||
# Requests held back by their own per-request delay wait
|
||||
# here instead of in their scope-set queue, so a not-yet-due request
|
||||
# never sits at a queue head and blocks the other requests that share
|
||||
# its scopes (the head-of-line blocking this scheduler exists to
|
||||
# avoid). Once the delay elapses, _promote_ready() moves the request
|
||||
# into its scope-set queue, where it competes normally.
|
||||
#
|
||||
# This is a min-heap of (deadline, seq, scope_set, request). heapq
|
||||
# needs a total order and always compares entries to keep its invariant;
|
||||
# seq (a monotonic counter) makes the (deadline, seq) prefix totally
|
||||
# ordered, so a deadline tie is broken there and the scope_set (a
|
||||
# frozenset, whose < is only the partial subset order) and the request
|
||||
# (no ordering at all) are never compared. The order among tied
|
||||
# deadlines is irrelevant beyond being deterministic.
|
||||
self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = []
|
||||
self._delayed_seq: int = 0
|
||||
|
||||
def _pqfactory(
|
||||
self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = ()
|
||||
) -> ScrapyPriorityQueue:
|
||||
return ScrapyPriorityQueue(
|
||||
self.crawler,
|
||||
self.downstream_queue_cls,
|
||||
self.key + "/" + _path_safe(_scope_set_key(scope_set)),
|
||||
startprios,
|
||||
start_queue_cls=self._start_queue_cls,
|
||||
)
|
||||
|
||||
def push(self, request: Request, scope_set: frozenset[ScopeID]) -> None:
|
||||
now = time.monotonic()
|
||||
self._promote_ready(now)
|
||||
delay = self._throttler.get_request_delay(request, now)
|
||||
if delay > 0:
|
||||
self._delayed_seq += 1
|
||||
heapq.heappush(
|
||||
self._delayed, (now + delay, self._delayed_seq, scope_set, request)
|
||||
)
|
||||
return
|
||||
self._push_to_queue(request, scope_set)
|
||||
|
||||
def _push_to_queue(self, request: Request, scope_set: frozenset[ScopeID]) -> None:
|
||||
if scope_set not in self.pqueues:
|
||||
self.pqueues[scope_set] = self._pqfactory(scope_set)
|
||||
self.pqueues[scope_set].push(request)
|
||||
|
||||
def _promote_ready(self, now: float) -> None:
|
||||
"""Move every held-back request whose per-request delay has elapsed into
|
||||
its scope-set queue, where it competes normally for its scopes."""
|
||||
while self._delayed and self._delayed[0][0] <= now:
|
||||
self._release_delayed(heapq.heappop(self._delayed))
|
||||
|
||||
def _release_delayed(
|
||||
self, entry: tuple[float, int, frozenset[ScopeID], Request]
|
||||
) -> None:
|
||||
_, _, scope_set, request = entry
|
||||
# The per-request delay has been honored (or the queue is closing), so
|
||||
# mark it consumed: the request must not be delayed again, and on resume
|
||||
# it must not re-block its scope set on a stale, no-longer-meaningful
|
||||
# deadline.
|
||||
request.meta["_throttler_delayed"] = True
|
||||
try:
|
||||
self._push_to_queue(request, scope_set)
|
||||
except ValueError as e:
|
||||
# A disk queue serializes on push; held-back requests defer that
|
||||
# serialization until here, so a non-serializable one would
|
||||
# otherwise raise while flushing on close and take the rest of the
|
||||
# disk queue down with it. Drop it with a warning instead, matching
|
||||
# how the scheduler handles unserializable requests at enqueue time.
|
||||
logger.warning(
|
||||
"Unable to serialize request: %(request)s - reason: %(reason)s",
|
||||
{"request": request, "reason": e},
|
||||
exc_info=True,
|
||||
extra={"spider": getattr(self.crawler, "spider", None)},
|
||||
)
|
||||
if self.crawler.stats is not None:
|
||||
self.crawler.stats.inc_value("scheduler/unserializable")
|
||||
|
||||
def _select(
|
||||
self,
|
||||
) -> tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None:
|
||||
"""Return the sendable ``(scope_set, queue)`` pair to pop from, or
|
||||
``None`` if no queue can be popped from right now.
|
||||
|
||||
Among the sendable queues (those whose scope set can be sent right now),
|
||||
the one whose head has the highest request priority is chosen; ties are
|
||||
broken by ascending load (the maximum
|
||||
:meth:`~scrapy.throttler.ThrottlerProtocol.get_scope_load` over the
|
||||
scopes of the queue), i.e. by preferring the least-busy scopes.
|
||||
"""
|
||||
self._promote_ready(time.monotonic())
|
||||
best_sort_key: tuple[int, float] | None = None
|
||||
best: tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None = None
|
||||
for scope_set, queue in self.pqueues.items():
|
||||
head = queue.peek()
|
||||
if head is None or not self._throttler.is_ready(head):
|
||||
continue
|
||||
load = max(
|
||||
(self._throttler.get_scope_load(scope_id) for scope_id in scope_set),
|
||||
default=0.0,
|
||||
)
|
||||
sort_key = (queue.priority(head), load)
|
||||
if best_sort_key is None or sort_key < best_sort_key:
|
||||
best_sort_key = sort_key
|
||||
best = (scope_set, queue)
|
||||
return best
|
||||
|
||||
def pop(self) -> Request | None:
|
||||
selected = self._select()
|
||||
if selected is None:
|
||||
return None
|
||||
scope_set, queue = selected
|
||||
request = queue.pop()
|
||||
if request is not None:
|
||||
self._throttler.reserve(request)
|
||||
if len(queue) == 0:
|
||||
del self.pqueues[scope_set]
|
||||
return request
|
||||
|
||||
def get_next_request_delay(self) -> float | None:
|
||||
now = time.monotonic()
|
||||
self._promote_ready(now)
|
||||
delay: float | None = None
|
||||
for queue in self.pqueues.values():
|
||||
head = queue.peek()
|
||||
if head is None:
|
||||
continue
|
||||
if self._throttler.is_ready(head):
|
||||
return 0.0
|
||||
head_delay = self._throttler.get_time_until_ready(head)
|
||||
if head_delay is None:
|
||||
continue
|
||||
if delay is None or head_delay < delay:
|
||||
delay = head_delay
|
||||
# A request held back only by its own delay is not in any
|
||||
# scope-set queue, so factor in when the earliest one is due.
|
||||
if self._delayed:
|
||||
next_delayed = max(0.0, self._delayed[0][0] - now)
|
||||
if delay is None or next_delayed < delay:
|
||||
delay = next_delayed
|
||||
return delay
|
||||
|
||||
def close(self) -> dict[str, list[int]]:
|
||||
# Flush held-back requests into their scope-set queues so they are
|
||||
# persisted (and restored on resume) rather than lost.
|
||||
while self._delayed:
|
||||
self._release_delayed(heapq.heappop(self._delayed))
|
||||
active = {
|
||||
_scope_set_key(scope_set): queue.close()
|
||||
for scope_set, queue in self.pqueues.items()
|
||||
}
|
||||
self.pqueues.clear()
|
||||
return active
|
||||
|
||||
def __len__(self) -> int:
|
||||
queued = sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
|
||||
return queued + len(self._delayed)
|
||||
|
|
|
|||
|
|
@ -67,6 +67,16 @@ class RobotParser(metaclass=ABCMeta):
|
|||
:type user_agent: str or bytes
|
||||
"""
|
||||
|
||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
||||
"""Return the ``Crawl-delay`` directive for ``user_agent`` as a number
|
||||
of seconds, or ``None`` if it is not set or the backend does not support
|
||||
it.
|
||||
|
||||
:param user_agent: User agent
|
||||
:type user_agent: str or bytes
|
||||
"""
|
||||
return None
|
||||
|
||||
|
||||
class PythonRobotParser(RobotParser):
|
||||
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
||||
|
|
@ -85,6 +95,10 @@ class PythonRobotParser(RobotParser):
|
|||
url = to_unicode(url)
|
||||
return self.rp.can_fetch(user_agent, url)
|
||||
|
||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
||||
delay = self.rp.crawl_delay(to_unicode(user_agent))
|
||||
return None if delay is None else float(delay)
|
||||
|
||||
|
||||
class RerpRobotParser(RobotParser):
|
||||
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
||||
|
|
@ -121,3 +135,7 @@ class ProtegoRobotParser(RobotParser):
|
|||
user_agent = to_unicode(user_agent)
|
||||
url = to_unicode(url)
|
||||
return self.rp.can_fetch(url, user_agent)
|
||||
|
||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
||||
delay = self.rp.crawl_delay(to_unicode(user_agent))
|
||||
return None if delay is None else float(delay)
|
||||
|
|
|
|||
|
|
@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]):
|
|||
:param default: the value to return if no setting is found
|
||||
:type default: object
|
||||
"""
|
||||
if name == "CONCURRENT_REQUESTS_PER_IP" and (
|
||||
isinstance(self[name], int) and self[name] != 0
|
||||
):
|
||||
warnings.warn(
|
||||
"The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
if name == "DNS_RESOLVER":
|
||||
warnings.warn(
|
||||
"The DNS_RESOLVER setting is deprecated, please use "
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to:
|
|||
import sys
|
||||
from importlib import import_module
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
__all__ = [
|
||||
"ADDONS",
|
||||
|
|
@ -33,6 +32,14 @@ __all__ = [
|
|||
"AWS_SESSION_TOKEN",
|
||||
"AWS_USE_SSL",
|
||||
"AWS_VERIFY",
|
||||
"BACKOFF_DELAY_FACTOR",
|
||||
"BACKOFF_ENABLED",
|
||||
"BACKOFF_EXCEPTIONS",
|
||||
"BACKOFF_HTTP_CODES",
|
||||
"BACKOFF_JITTER",
|
||||
"BACKOFF_MAX_DELAY",
|
||||
"BACKOFF_MIN_DELAY",
|
||||
"BACKOFF_WINDOW",
|
||||
"BOT_NAME",
|
||||
"CLOSESPIDER_ERRORCOUNT",
|
||||
"CLOSESPIDER_ITEMCOUNT",
|
||||
|
|
@ -170,6 +177,7 @@ __all__ = [
|
|||
"RANDOMIZE_DOWNLOAD_DELAY",
|
||||
"REACTOR_THREADPOOL_MAXSIZE",
|
||||
"REDIRECT_ENABLED",
|
||||
"REDIRECT_MAX_DELAY",
|
||||
"REDIRECT_MAX_TIMES",
|
||||
"REDIRECT_PRIORITY_ADJUST",
|
||||
"REFERER_ENABLED",
|
||||
|
|
@ -209,6 +217,16 @@ __all__ = [
|
|||
"TELNETCONSOLE_PORT",
|
||||
"TELNETCONSOLE_USERNAME",
|
||||
"TEMPLATES_DIR",
|
||||
"THROTTLER",
|
||||
"THROTTLER_DEBUG",
|
||||
"THROTTLER_ROBOTSTXT_MAX_DELAY",
|
||||
"THROTTLER_ROBOTSTXT_OBEY",
|
||||
"THROTTLER_SCOPES",
|
||||
"THROTTLER_SCOPE_CONCURRENCY",
|
||||
"THROTTLER_SCOPE_LIMIT",
|
||||
"THROTTLER_SCOPE_MANAGER",
|
||||
"THROTTLER_SCOPE_MAX_IDLE",
|
||||
"THROTTLER_WINDOW",
|
||||
"TWISTED_DNS_RESOLVER",
|
||||
"TWISTED_REACTOR",
|
||||
"TWISTED_REACTOR_ENABLED",
|
||||
|
|
@ -235,6 +253,19 @@ AWS_SESSION_TOKEN = None
|
|||
AWS_USE_SSL = None
|
||||
AWS_VERIFY = None
|
||||
|
||||
BACKOFF_DELAY_FACTOR = 2.0
|
||||
BACKOFF_ENABLED = True
|
||||
BACKOFF_EXCEPTIONS = [
|
||||
"scrapy.exceptions.DownloadFailedError",
|
||||
"scrapy.exceptions.DownloadTimeoutError",
|
||||
"scrapy.exceptions.ResponseDataLossError",
|
||||
]
|
||||
BACKOFF_HTTP_CODES = [429, 502, 503, 504, 520, 521, 522, 523, 524]
|
||||
BACKOFF_JITTER = 0.1
|
||||
BACKOFF_MAX_DELAY = 300.0
|
||||
BACKOFF_MIN_DELAY = 1.0
|
||||
BACKOFF_WINDOW = 60.0
|
||||
|
||||
BOT_NAME = "scrapybot"
|
||||
|
||||
CLOSESPIDER_ERRORCOUNT = 0
|
||||
|
|
@ -324,6 +355,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
|
|||
"scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
|
||||
"scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
|
||||
"scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
|
||||
"scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 650,
|
||||
"scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700,
|
||||
"scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750,
|
||||
"scrapy.downloadermiddlewares.stats.DownloaderStats": 850,
|
||||
|
|
@ -493,6 +525,7 @@ RANDOMIZE_DOWNLOAD_DELAY = True
|
|||
REACTOR_THREADPOOL_MAXSIZE = 10
|
||||
|
||||
REDIRECT_ENABLED = True
|
||||
REDIRECT_MAX_DELAY = 120.0
|
||||
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
||||
REDIRECT_PRIORITY_ADJUST = +2
|
||||
|
||||
|
|
@ -574,9 +607,21 @@ TELNETCONSOLE_PASSWORD = None
|
|||
|
||||
TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
|
||||
|
||||
THROTTLER = "scrapy.throttler.Throttler"
|
||||
THROTTLER_SCOPE_MANAGER = "scrapy.throttler.ThrottlerScopeManager"
|
||||
THROTTLER_SCOPES = {}
|
||||
THROTTLER_WINDOW = 60.0
|
||||
THROTTLER_ROBOTSTXT_OBEY = True
|
||||
THROTTLER_ROBOTSTXT_MAX_DELAY = 60.0
|
||||
THROTTLER_SCOPE_CONCURRENCY = 1
|
||||
THROTTLER_SCOPE_LIMIT = 100000
|
||||
THROTTLER_SCOPE_MAX_IDLE = 3600.0
|
||||
THROTTLER_DEBUG = False
|
||||
|
||||
TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"
|
||||
|
||||
TWISTED_REACTOR_ENABLED = True
|
||||
|
||||
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
|
||||
|
||||
URLLENGTH_LIMIT = 2083
|
||||
|
|
@ -584,19 +629,3 @@ URLLENGTH_LIMIT = 2083
|
|||
USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)"
|
||||
|
||||
WARN_ON_GENERATOR_RETURN_VALUE = True
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
if name == "CONCURRENT_REQUESTS_PER_IP":
|
||||
import warnings # noqa: PLC0415
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415
|
||||
|
||||
warnings.warn(
|
||||
"The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
return 0
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ response_received = object()
|
|||
response_downloaded = object()
|
||||
headers_received = object()
|
||||
bytes_received = object()
|
||||
robots_parsed = object()
|
||||
item_scraped = object()
|
||||
item_dropped = object()
|
||||
item_error = object()
|
||||
|
|
|
|||
|
|
@ -21,9 +21,8 @@ ADDONS = {}
|
|||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = True
|
||||
|
||||
# Concurrency and throttling settings
|
||||
#CONCURRENT_REQUESTS = 16
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN = 1
|
||||
THROTTLER_SCOPE_CONCURRENCY = 1
|
||||
DOWNLOAD_DELAY = 1
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
|
|
@ -62,19 +61,6 @@ DOWNLOAD_DELAY = 1
|
|||
# "$project_name.pipelines.${ProjectName}Pipeline": 300,
|
||||
#}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,49 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.http import Response
|
||||
|
||||
|
||||
def _decoded_header(response: Response, name: str) -> str | None:
|
||||
"""Return the stripped UTF-8 value of the *name* header of *response*, or
|
||||
``None`` if it is absent or not valid UTF-8."""
|
||||
raw = response.headers.get(name)
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return raw.decode("utf-8").strip()
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_retry_after(response: Response) -> float | None:
|
||||
value = _decoded_header(response, "Retry-After")
|
||||
if value is None:
|
||||
return None
|
||||
if value.isdigit():
|
||||
return float(value) # seconds
|
||||
try:
|
||||
date = parsedate_to_datetime(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
if date.tzinfo is None:
|
||||
date = date.replace(tzinfo=dt.timezone.utc)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
seconds_to_wait = (date - now).total_seconds()
|
||||
# Keep sub-second precision (a date less than a second away must not be
|
||||
# truncated to 0 and dropped); a past or present date yields no delay.
|
||||
return max(0.0, seconds_to_wait) or None
|
||||
|
||||
|
||||
def _parse_ratelimit_reset(response: Response) -> float | None:
|
||||
value = _decoded_header(response, "RateLimit-Reset")
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
|
@ -5,11 +5,11 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.internet.task import LoopingCall
|
||||
from twisted.internet.task import LoopingCall, deferLater
|
||||
from twisted.internet.threads import deferToThread
|
||||
|
||||
from scrapy.utils.asyncgen import as_async_generator
|
||||
|
|
@ -311,3 +311,106 @@ async def run_in_thread(
|
|||
from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415
|
||||
|
||||
return await maybe_deferred_to_future(deferToThread(func, *args, **kwargs))
|
||||
|
||||
|
||||
async def sleep(seconds: float) -> None:
|
||||
"""Sleep for *seconds*, working in asyncio-reactor, non-asyncio-reactor,
|
||||
and reactorless modes.
|
||||
|
||||
Uses :func:`asyncio.sleep` when asyncio is available, and
|
||||
:func:`~twisted.internet.task.deferLater` otherwise.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
"""
|
||||
if is_asyncio_available():
|
||||
await asyncio.sleep(seconds)
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
|
||||
# circular import
|
||||
from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415
|
||||
|
||||
await maybe_deferred_to_future(deferLater(reactor, seconds, lambda: None))
|
||||
|
||||
|
||||
async def wait_for_first(
|
||||
deferreds: Sequence[Deferred[Any]],
|
||||
*,
|
||||
timeout: float | None = None,
|
||||
) -> tuple[set[Deferred[Any]], set[Deferred[Any]]]:
|
||||
"""Wait for the first of *deferreds* to fire, or until *timeout* seconds pass.
|
||||
|
||||
Returns ``(done, pending)`` — two sets partitioning the input
|
||||
:class:`~twisted.internet.defer.Deferred` objects — mirroring the API of
|
||||
:func:`asyncio.wait`.
|
||||
|
||||
Unfired deferreds in the ``pending`` set are neither cancelled nor
|
||||
otherwise modified; the caller is responsible for any cleanup.
|
||||
|
||||
A deferred that fails counts as done and its failure is **not** re-raised
|
||||
here (it stays on the deferred for the caller to inspect or handle), exactly
|
||||
as a failed awaitable lands in the ``done`` set of :func:`asyncio.wait`.
|
||||
|
||||
Returns ``(set(), set())`` immediately when *deferreds* is empty.
|
||||
|
||||
Works transparently in asyncio-reactor, non-asyncio-reactor, and
|
||||
reactorless modes.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
"""
|
||||
if not deferreds:
|
||||
return set(), set()
|
||||
|
||||
if is_asyncio_available():
|
||||
# circular import
|
||||
from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415
|
||||
|
||||
future_to_deferred = {maybe_deferred_to_future(d): d for d in deferreds}
|
||||
done_futures, pending_futures = await asyncio.wait(
|
||||
list(future_to_deferred),
|
||||
timeout=timeout,
|
||||
return_when=asyncio.FIRST_COMPLETED,
|
||||
)
|
||||
return (
|
||||
{future_to_deferred[f] for f in done_futures},
|
||||
{future_to_deferred[f] for f in pending_futures},
|
||||
)
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
# circular import
|
||||
from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415
|
||||
|
||||
# Fire a single signal Deferred as soon as any input Deferred (or the
|
||||
# timeout) completes, whether it succeeds or fails. Unlike a DeferredList
|
||||
# with fireOnOneErrback (which would raise the first failure into this
|
||||
# coroutine), _fire passes each result through untouched, so a failure just
|
||||
# marks its Deferred as done and stays there for the caller — matching the
|
||||
# asyncio branch and asyncio.wait(return_when=FIRST_COMPLETED).
|
||||
signal: Deferred[None] = Deferred()
|
||||
|
||||
def _fire(result: Any) -> Any:
|
||||
if not signal.called:
|
||||
signal.callback(None)
|
||||
return result
|
||||
|
||||
for d in deferreds:
|
||||
d.addBoth(_fire)
|
||||
|
||||
timeout_deferred: Deferred[Any] | None = None
|
||||
if timeout is not None:
|
||||
timeout_deferred = deferLater(reactor, timeout, lambda: None)
|
||||
timeout_deferred.addBoth(_fire)
|
||||
# Swallow the CancelledError from the finally-block cancel() below.
|
||||
timeout_deferred.addErrback(lambda _: None)
|
||||
|
||||
try:
|
||||
await maybe_deferred_to_future(signal)
|
||||
finally:
|
||||
if timeout_deferred is not None and not timeout_deferred.called:
|
||||
timeout_deferred.cancel()
|
||||
|
||||
return (
|
||||
{d for d in deferreds if d.called},
|
||||
{d for d in deferreds if not d.called},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any:
|
|||
return obj
|
||||
|
||||
|
||||
def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]:
|
||||
"""Resolve *objects* (objects or import paths) to a tuple of objects."""
|
||||
return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects)
|
||||
|
||||
|
||||
def walk_modules_iter(path: str) -> Iterable[ModuleType]:
|
||||
"""Loads a module and all its submodules from the given module path and
|
||||
returns them. If *any* module throws an exception while importing, that
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from twisted.web.client import Agent
|
|||
|
||||
from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.settings import default_settings
|
||||
from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
|
||||
|
|
@ -66,13 +67,19 @@ def get_crawler(
|
|||
will be used to populate the crawler settings with a project level
|
||||
priority.
|
||||
"""
|
||||
# When needed, useful settings can be added here, e.g. ones that prevent
|
||||
# deprecation warnings.
|
||||
settings: dict[str, Any] = {
|
||||
"TELNETCONSOLE_ENABLED": False,
|
||||
**get_reactor_settings(),
|
||||
**(settings_dict or {}),
|
||||
}
|
||||
if prevent_warnings:
|
||||
# Pin the per-scope concurrency to CONCURRENT_REQUESTS_PER_DOMAIN's
|
||||
# default: keeps the suite's historical behavior and silences the
|
||||
# warn-then-flip warning (see throttler._warn_on_deprecated_concurrency).
|
||||
settings.setdefault(
|
||||
"THROTTLER_SCOPE_CONCURRENCY",
|
||||
default_settings.CONCURRENT_REQUESTS_PER_DOMAIN,
|
||||
)
|
||||
runner: CrawlerRunnerBase
|
||||
if is_reactor_installed():
|
||||
runner = CrawlerRunner(settings)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from twisted.web import server, static
|
|||
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
|
||||
from twisted.web.client import Response as TxResponse
|
||||
|
||||
from scrapy.core.downloader import Downloader, Slot, tls
|
||||
from scrapy.core.downloader import Downloader, Slot, _Slot, tls
|
||||
from scrapy.core.downloader.contextfactory import (
|
||||
_load_context_factory_from_settings,
|
||||
_ScrapyClientContextFactory,
|
||||
|
|
@ -41,8 +41,20 @@ if TYPE_CHECKING:
|
|||
|
||||
class TestSlot:
|
||||
def test_repr(self):
|
||||
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
|
||||
assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)"
|
||||
slot = _Slot()
|
||||
assert repr(slot) == "_Slot()"
|
||||
|
||||
def test_deprecated(self):
|
||||
with pytest.warns(ScrapyDeprecationWarning, match="Slot class is deprecated"):
|
||||
Slot()
|
||||
|
||||
def test_deprecated_subclass(self):
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match="inherits from the deprecated Slot"
|
||||
):
|
||||
|
||||
class MySlot(Slot):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.requires_reactor # this test is related to the Twisted HTTP code
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logging
|
|||
import re
|
||||
import signal
|
||||
import threading
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, ClassVar
|
||||
from unittest.mock import MagicMock
|
||||
|
|
@ -52,6 +53,77 @@ def get_raw_crawler(
|
|||
return Crawler(spidercls or DefaultSpider, settings)
|
||||
|
||||
|
||||
class TestScopeConcurrencyBridge:
|
||||
"""CONCURRENT_REQUESTS_PER_DOMAIN is deprecated and bridged into
|
||||
THROTTLER_SCOPE_CONCURRENCY by setting priority (see
|
||||
scrapy.throttler._default_scope_concurrency)."""
|
||||
|
||||
@staticmethod
|
||||
def _resolve(settings: Settings) -> int:
|
||||
from scrapy.throttler import _default_scope_concurrency # noqa: PLC0415
|
||||
|
||||
return _default_scope_concurrency(settings)
|
||||
|
||||
def test_neither_set_preserves_per_domain_default(self) -> None:
|
||||
# Neither setting overridden: keep the historical per-domain default.
|
||||
assert self._resolve(Settings()) == 8
|
||||
|
||||
def test_per_domain_overrides(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project")
|
||||
assert self._resolve(settings) == 4
|
||||
|
||||
def test_scope_concurrency_overrides(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("THROTTLER_SCOPE_CONCURRENCY", 4, priority="project")
|
||||
assert self._resolve(settings) == 4
|
||||
|
||||
def test_new_setting_wins_on_non_default_tie(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project")
|
||||
settings.set("THROTTLER_SCOPE_CONCURRENCY", 9, priority="project")
|
||||
assert self._resolve(settings) == 9
|
||||
|
||||
def test_higher_priority_deprecated_setting_wins(self) -> None:
|
||||
settings = Settings()
|
||||
settings.set("THROTTLER_SCOPE_CONCURRENCY", 9, priority="project")
|
||||
settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="cmdline")
|
||||
assert self._resolve(settings) == 4
|
||||
|
||||
def test_warns_flip_when_neither_set(self) -> None:
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="Once CONCURRENT_REQUESTS_PER_DOMAIN is removed",
|
||||
):
|
||||
get_crawler(prevent_warnings=False)
|
||||
|
||||
def test_warns_deprecated_when_per_domain_set(self) -> None:
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated",
|
||||
):
|
||||
get_crawler(
|
||||
settings_dict={"CONCURRENT_REQUESTS_PER_DOMAIN": 4},
|
||||
prevent_warnings=False,
|
||||
)
|
||||
|
||||
def test_no_concurrency_warning_when_scope_set(self) -> None:
|
||||
with warnings.catch_warnings(record=True) as recorded:
|
||||
warnings.simplefilter("always")
|
||||
get_crawler(
|
||||
settings_dict={"THROTTLER_SCOPE_CONCURRENCY": 4},
|
||||
prevent_warnings=False,
|
||||
)
|
||||
messages = [str(w.message) for w in recorded]
|
||||
assert not any(
|
||||
"Once CONCURRENT_REQUESTS_PER_DOMAIN is removed" in m for m in messages
|
||||
)
|
||||
assert not any(
|
||||
"CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated" in m
|
||||
for m in messages
|
||||
)
|
||||
|
||||
|
||||
class TestBaseCrawler:
|
||||
@staticmethod
|
||||
def assertOptionIsDefault(settings: Settings, key: str) -> None:
|
||||
|
|
@ -87,6 +159,30 @@ class TestCrawler(TestBaseCrawler):
|
|||
assert not settings.frozen
|
||||
assert crawler.settings.frozen
|
||||
|
||||
def test_spider_download_delay_deprecated(self) -> None:
|
||||
class DelaySpider(DefaultSpider):
|
||||
download_delay = 2.5
|
||||
|
||||
crawler = Crawler(DelaySpider)
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match="'download_delay' spider attribute"
|
||||
):
|
||||
crawler._apply_settings()
|
||||
assert crawler.settings.getfloat("DOWNLOAD_DELAY") == 2.5
|
||||
|
||||
def test_spider_download_delay_overridden_by_setting(self) -> None:
|
||||
class DelaySpider(DefaultSpider):
|
||||
download_delay = 2.5
|
||||
|
||||
crawler = Crawler(DelaySpider)
|
||||
crawler.settings.set("DOWNLOAD_DELAY", 5.0, priority="spider")
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning,
|
||||
match="'download_delay' spider attribute.*being ignored",
|
||||
):
|
||||
crawler._apply_settings()
|
||||
assert crawler.settings.getfloat("DOWNLOAD_DELAY") == 5.0
|
||||
|
||||
def test_crawler_accepts_dict(self) -> None:
|
||||
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
|
||||
assert crawler.settings["foo"] == "bar"
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import pytest
|
|||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python import failure
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
|
||||
from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
|
||||
from scrapy.http import Request, Response, TextResponse
|
||||
|
|
@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware:
|
|||
self.crawler: mock.MagicMock = mock.MagicMock()
|
||||
self.crawler.settings = Settings()
|
||||
self.crawler.engine.download_async = mock.AsyncMock()
|
||||
self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])
|
||||
|
||||
def teardown_method(self):
|
||||
del self.crawler
|
||||
|
|
@ -74,6 +76,22 @@ Disallow: /some/randome/page.html
|
|||
Request("http://site.local/wiki/Käyttäjä:"), middleware
|
||||
)
|
||||
|
||||
@coroutine_test
|
||||
async def test_robotstxt_emits_robots_parsed_signal(self):
|
||||
crawler = self._get_successful_crawler()
|
||||
middleware = RobotsTxtMiddleware(crawler)
|
||||
request = Request("http://site.local/allowed")
|
||||
await self.assertNotIgnored(request, middleware)
|
||||
calls = [
|
||||
kwargs
|
||||
for _, kwargs in crawler.signals.send_catch_log_async.call_args_list
|
||||
if kwargs.get("signal") is signals.robots_parsed
|
||||
]
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["request"] is request
|
||||
assert calls[0]["robotparser"] is not None
|
||||
assert "crawler" not in calls[0]
|
||||
|
||||
@coroutine_test
|
||||
async def test_robotstxt_multiple_reqs(self) -> None:
|
||||
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
|
||||
|
|
|
|||
|
|
@ -1,39 +1,32 @@
|
|||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.core.downloader import Downloader, Slot
|
||||
from scrapy.core.downloader import Downloader
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.spider import DefaultSpider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver.http import MockServer
|
||||
from tests.spiders import MetaSpider
|
||||
from tests.utils.decorators import coroutine_test, inline_callbacks_test
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
|
||||
class DownloaderSlotsSettingsTestSpider(MetaSpider):
|
||||
name = "downloader_slots"
|
||||
|
||||
custom_settings = {
|
||||
"DOWNLOAD_DELAY": 1,
|
||||
"RANDOMIZE_DOWNLOAD_DELAY": False,
|
||||
"DOWNLOAD_SLOTS": {
|
||||
"quotes.toscrape.com": {
|
||||
"concurrency": 1,
|
||||
"delay": 2,
|
||||
"randomize_delay": False,
|
||||
"throttle": False,
|
||||
},
|
||||
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
|
||||
"quotes.toscrape.com": {"concurrency": 1},
|
||||
"books.toscrape.com": {"concurrency": 2},
|
||||
},
|
||||
}
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert self.mockserver
|
||||
self.default_slot = self.mockserver.host
|
||||
self.default_slot = urlparse(self.mockserver.url("/")).netloc
|
||||
self.times: dict[str, list[float]] = {}
|
||||
|
||||
async def start(self):
|
||||
|
|
@ -45,65 +38,84 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
|
|||
|
||||
def parse(self, response):
|
||||
slot = response.meta.get("download_slot", self.default_slot)
|
||||
self.times[slot].append(time.time())
|
||||
self.times[slot].append(response.meta.get("download_latency"))
|
||||
url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
|
||||
yield Request(url, callback=self.not_parse, meta={"download_slot": slot})
|
||||
|
||||
def not_parse(self, response):
|
||||
slot = response.meta.get("download_slot", self.default_slot)
|
||||
self.times[slot].append(time.time())
|
||||
|
||||
|
||||
class TestCrawl:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.mockserver = MockServer()
|
||||
cls.mockserver.__enter__()
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
cls.mockserver.__exit__(None, None, None)
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_delay(self):
|
||||
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
slots = crawler.engine.downloader.slots
|
||||
times = crawler.spider.times
|
||||
tolerance = 0.3
|
||||
|
||||
delays_real = {k: v[1] - v[0] for k, v in times.items()}
|
||||
error_delta = {
|
||||
k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
|
||||
for k, v in slots.items()
|
||||
}
|
||||
|
||||
assert max(list(error_delta.values())) < tolerance
|
||||
self.times[slot].append(response.meta.get("download_latency"))
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_params():
|
||||
params = {
|
||||
"concurrency": 1,
|
||||
"delay": 2,
|
||||
"randomize_delay": False,
|
||||
}
|
||||
settings = {
|
||||
"DOWNLOAD_SLOTS": {
|
||||
"example.com": params,
|
||||
},
|
||||
}
|
||||
async def test_concurrency_key_deprecated():
|
||||
settings = {"DOWNLOAD_SLOTS": {"example.com": {"concurrency": 3}}}
|
||||
crawler = get_crawler(DefaultSpider, settings_dict=settings)
|
||||
crawler.spider = crawler._create_spider()
|
||||
with pytest.warns(ScrapyDeprecationWarning) as warns:
|
||||
downloader = Downloader(crawler)
|
||||
messages = [str(w.message) for w in warns]
|
||||
assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages)
|
||||
downloader._get_slot(Request("https://example.com"))
|
||||
downloader.close()
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_slots_deprecated():
|
||||
settings = {"DOWNLOAD_SLOTS": {"example.com": {"concurrency": 2}}}
|
||||
crawler = get_crawler(DefaultSpider, settings_dict=settings)
|
||||
crawler.spider = crawler._create_spider()
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match="DOWNLOAD_SLOTS setting is deprecated"
|
||||
):
|
||||
Downloader(crawler).close()
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_slots_deprecated():
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
crawler.spider = crawler._create_spider()
|
||||
downloader = Downloader(crawler)
|
||||
request = Request("https://example.com")
|
||||
_, actual = downloader._get_slot(request)
|
||||
request.meta[Downloader.DOWNLOAD_SLOT] = "example.com"
|
||||
downloader.active.add(request)
|
||||
with pytest.warns(ScrapyDeprecationWarning, match="Downloader.slots is deprecated"):
|
||||
slot = downloader.slots.get("example.com")
|
||||
assert slot is not None
|
||||
assert isinstance(slot.active, set)
|
||||
assert request in slot.active
|
||||
downloader.active.discard(request)
|
||||
downloader.close()
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_slot_meta_deprecated():
|
||||
crawler = get_crawler(DefaultSpider)
|
||||
crawler.spider = crawler._create_spider()
|
||||
downloader = Downloader(crawler)
|
||||
request = Request("https://example.com")
|
||||
request.meta["download_slot"] = "custom"
|
||||
with pytest.warns(
|
||||
ScrapyDeprecationWarning, match="'download_slot' request meta key is deprecated"
|
||||
):
|
||||
key, _ = downloader._get_slot(request)
|
||||
downloader.close()
|
||||
assert key == "custom"
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_delay_deprecated():
|
||||
settings = {
|
||||
"DOWNLOAD_SLOTS": {"example.com": {"delay": 2, "randomize_delay": False}}
|
||||
}
|
||||
crawler = get_crawler(DefaultSpider, settings_dict=settings)
|
||||
crawler.spider = crawler._create_spider()
|
||||
with pytest.warns(ScrapyDeprecationWarning) as warns:
|
||||
downloader = Downloader(crawler)
|
||||
messages = [str(w.message) for w in warns]
|
||||
assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages)
|
||||
downloader._get_slot(Request("https://example.com"))
|
||||
downloader.close()
|
||||
expected = Slot(**params)
|
||||
for param in params:
|
||||
assert getattr(expected, param) == getattr(actual, param), (
|
||||
f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}"
|
||||
)
|
||||
|
||||
|
||||
@coroutine_test
|
||||
|
|
@ -122,7 +134,7 @@ async def test_get_slot_deprecated_spider_arg():
|
|||
downloader.close()
|
||||
|
||||
assert key1 == key2
|
||||
assert slot1 == slot2
|
||||
assert slot1._key == slot2._key
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -132,6 +144,7 @@ async def test_get_slot_deprecated_spider_arg():
|
|||
"scrapy.pqueues.DownloaderAwarePriorityQueue",
|
||||
],
|
||||
)
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
@coroutine_test
|
||||
async def test_none_slot_with_priority_queue(
|
||||
mockserver: MockServer, priority_queue_class: str
|
||||
|
|
|
|||
|
|
@ -595,6 +595,21 @@ class TestEngineDownloadAsync:
|
|||
engine._slot.add_request.assert_called_once_with(request)
|
||||
engine._slot.remove_request.assert_called_once_with(request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_fetch_needs_spider(self, engine):
|
||||
"""A downloader whose fetch() requires a spider gets it passed in."""
|
||||
engine._downloader_fetch_needs_spider = True
|
||||
request = Request("http://example.com")
|
||||
response = Response("http://example.com", body=b"test body")
|
||||
engine.spider = Mock()
|
||||
engine.downloader.fetch.return_value = defer.succeed(response)
|
||||
engine._slot.add_request = Mock()
|
||||
engine._slot.remove_request = Mock()
|
||||
|
||||
result = await self._download(engine, request)
|
||||
assert result == response
|
||||
engine.downloader.fetch.assert_called_once_with(request, engine.spider)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestEngineDownload(TestEngineDownloadAsync):
|
||||
|
|
@ -640,6 +655,154 @@ async def test_request_scheduled_signal():
|
|||
crawler.signals.disconnect(signal_handler, signals.request_scheduled)
|
||||
|
||||
|
||||
class TestEngineThrottler:
|
||||
@pytest.fixture
|
||||
def engine(self):
|
||||
crawler = get_crawler(MySpider)
|
||||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
yield engine
|
||||
engine.downloader.close()
|
||||
|
||||
def test_pause_cancels_delay_wakeup(self, engine):
|
||||
wakeup = Mock()
|
||||
engine._delay_wakeup = wakeup
|
||||
engine.pause()
|
||||
assert engine.paused is True
|
||||
wakeup.cancel.assert_called_once_with()
|
||||
assert engine._delay_wakeup is None
|
||||
engine.unpause()
|
||||
assert engine.paused is False
|
||||
|
||||
@pytest.mark.requires_reactor # call_later() needs a reactor or asyncio loop
|
||||
def test_maybe_arm_delay_wakeup_arms_timer(self, engine):
|
||||
engine._get_next_request_delay = lambda: 5.0
|
||||
engine._slot = Mock()
|
||||
engine._slot.scheduler.has_pending_requests.return_value = True
|
||||
engine._maybe_arm_delay_wakeup()
|
||||
assert engine._delay_wakeup is not None
|
||||
# Cancel the scheduled reactor call so it does not leak into other tests.
|
||||
engine._cancel_delay_wakeup()
|
||||
|
||||
def test_maybe_arm_delay_wakeup_no_delay(self, engine):
|
||||
engine._get_next_request_delay = lambda: None
|
||||
engine._slot = Mock()
|
||||
engine._slot.scheduler.has_pending_requests.return_value = True
|
||||
engine._maybe_arm_delay_wakeup()
|
||||
assert engine._delay_wakeup is None
|
||||
|
||||
def test_maybe_arm_delay_wakeup_zero_delay(self, engine):
|
||||
# A 0 delay means a request is ready but could not be sent (e.g. the
|
||||
# downloader is at capacity); arming a 0-second timer would busy-loop
|
||||
# the engine, so no timer must be armed.
|
||||
engine._get_next_request_delay = lambda: 0.0
|
||||
engine._slot = Mock()
|
||||
engine._slot.scheduler.has_pending_requests.return_value = True
|
||||
engine._maybe_arm_delay_wakeup()
|
||||
assert engine._delay_wakeup is None
|
||||
|
||||
def test_maybe_arm_delay_wakeup_not_supported(self, engine):
|
||||
# A scheduler without get_next_request_delay never arms a timer.
|
||||
engine._get_next_request_delay = None
|
||||
engine._slot = Mock()
|
||||
engine._slot.scheduler.has_pending_requests.return_value = True
|
||||
engine._maybe_arm_delay_wakeup()
|
||||
assert engine._delay_wakeup is None
|
||||
|
||||
def test_maybe_warn_throttler_backout(self, engine):
|
||||
# A scheduler without get_next_request_delay is not throttler-aware, so
|
||||
# the warning recommends switching to one.
|
||||
engine._get_next_request_delay = None
|
||||
with LogCapture() as log:
|
||||
engine._maybe_warn_throttler_backout()
|
||||
# A second call is a no-op (the warning is emitted only once).
|
||||
engine._maybe_warn_throttler_backout()
|
||||
assert engine._throttler_backout_warned is True
|
||||
assert str(log).count("ThrottlerAwareScheduler") == 1
|
||||
|
||||
def test_maybe_warn_throttler_backout_throttler_aware(self, engine):
|
||||
# A throttler-aware scheduler (one with get_next_request_delay) holds
|
||||
# throttled requests itself, so no warning is emitted.
|
||||
engine._get_next_request_delay = lambda: None
|
||||
with LogCapture() as log:
|
||||
engine._maybe_warn_throttler_backout()
|
||||
assert engine._throttler_backout_warned is False
|
||||
assert "ThrottlerAwareScheduler" not in str(log)
|
||||
|
||||
def test_spider_is_idle_false_while_scheduling(self, engine):
|
||||
engine._slot = Mock()
|
||||
engine.scraper.slot = Mock()
|
||||
engine.scraper.slot.is_idle.return_value = True
|
||||
engine.downloader = Mock()
|
||||
engine.downloader.active = []
|
||||
engine._throttler_waiting = set()
|
||||
engine._start = None
|
||||
engine._scheduling = 1
|
||||
# An in-flight async enqueue keeps the spider from being considered idle.
|
||||
assert engine.spider_is_idle() is False
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_request_async_dropped(self, engine):
|
||||
scheduler = Mock()
|
||||
|
||||
async def enqueue_request_async(request):
|
||||
return False
|
||||
|
||||
scheduler.enqueue_request_async = enqueue_request_async
|
||||
engine._slot = Mock()
|
||||
engine._slot.scheduler = scheduler
|
||||
engine.spider = Mock()
|
||||
dropped = []
|
||||
|
||||
def on_dropped(request, spider):
|
||||
dropped.append(request)
|
||||
|
||||
engine.signals.connect(on_dropped, signals.request_dropped, weak=False)
|
||||
engine._scheduling = 1
|
||||
request = Request("http://a.example")
|
||||
await engine._enqueue_request_async(request)
|
||||
assert dropped == [request]
|
||||
assert engine._scheduling == 0
|
||||
engine._slot.nextcall.schedule.assert_called_once_with()
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_request_async_error(self, engine):
|
||||
scheduler = Mock()
|
||||
|
||||
async def enqueue_request_async(request):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
scheduler.enqueue_request_async = enqueue_request_async
|
||||
engine._slot = Mock()
|
||||
engine._slot.scheduler = scheduler
|
||||
engine.spider = Mock()
|
||||
engine._scheduling = 1
|
||||
with LogCapture() as log:
|
||||
await engine._enqueue_request_async(Request("http://a.example"))
|
||||
assert "Error while enqueuing request" in str(log)
|
||||
assert engine._scheduling == 0
|
||||
engine._slot.nextcall.schedule.assert_called_once_with()
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_request_async_slot_gone(self, engine):
|
||||
scheduler = Mock()
|
||||
|
||||
async def enqueue_request_async(request):
|
||||
# The spider is closed while the enqueue is in flight.
|
||||
engine._slot = None
|
||||
return True
|
||||
|
||||
scheduler.enqueue_request_async = enqueue_request_async
|
||||
slot = Mock()
|
||||
slot.scheduler = scheduler
|
||||
engine._slot = slot
|
||||
engine.spider = Mock()
|
||||
engine._scheduling = 1
|
||||
await engine._enqueue_request_async(Request("http://a.example"))
|
||||
assert engine._scheduling == 0
|
||||
# No reschedule is attempted once the slot is gone.
|
||||
slot.nextcall.schedule.assert_not_called()
|
||||
|
||||
|
||||
class TestEngineCloseSpider:
|
||||
"""Tests for exception handling coverage during close_spider_async()."""
|
||||
|
||||
|
|
@ -688,7 +851,7 @@ class TestEngineCloseSpider:
|
|||
engine = ExecutionEngine(crawler, lambda _: None)
|
||||
crawler.engine = engine
|
||||
await engine.open_spider_async()
|
||||
del engine.downloader.slots
|
||||
engine.downloader.close = Mock(side_effect=Exception("close failed"))
|
||||
await engine.close_spider_async()
|
||||
assert "Downloader close failure" in caplog.text
|
||||
|
||||
|
|
|
|||
|
|
@ -4,14 +4,19 @@ from unittest.mock import Mock
|
|||
import pytest
|
||||
import queuelib
|
||||
|
||||
from scrapy.core.downloader import Downloader
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue
|
||||
from scrapy.pqueues import (
|
||||
DownloaderAwarePriorityQueue,
|
||||
ScrapyPriorityQueue,
|
||||
ThrottlerAwarePriorityQueue,
|
||||
)
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
|
||||
from scrapy.throttler import iter_scopes
|
||||
from scrapy.utils.misc import build_from_crawler, load_object
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.test_scheduler import MockDownloader
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
|
||||
class TestPriorityQueue:
|
||||
|
|
@ -186,22 +191,22 @@ class TestDownloaderAwarePriorityQueue:
|
|||
# No active downloads are tracked in the downloader, so every slot has
|
||||
# the same score and tie-breaking must not starve a slot.
|
||||
req_a1 = Request("https://example.org/a1")
|
||||
req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
|
||||
req_a1.meta["throttler_scopes"] = "slot-a"
|
||||
req_b1 = Request("https://example.org/b1")
|
||||
req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
|
||||
req_b1.meta["throttler_scopes"] = "slot-b"
|
||||
req_a2 = Request("https://example.org/a2")
|
||||
req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
|
||||
req_a2.meta["throttler_scopes"] = "slot-a"
|
||||
req_b2 = Request("https://example.org/b2")
|
||||
req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
|
||||
req_b2.meta["throttler_scopes"] = "slot-b"
|
||||
|
||||
for request in (req_a1, req_b1, req_a2, req_b2):
|
||||
self.queue.push(request)
|
||||
|
||||
slots = [
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
]
|
||||
|
||||
assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"]
|
||||
|
|
@ -210,48 +215,49 @@ class TestDownloaderAwarePriorityQueue:
|
|||
# If the selected slot becomes empty, rotation should continue from
|
||||
# that slot marker to avoid restarting from the smallest slot.
|
||||
req_a1 = Request("https://example.org/a1")
|
||||
req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
|
||||
req_a1.meta["throttler_scopes"] = "slot-a"
|
||||
req_a2 = Request("https://example.org/a2")
|
||||
req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
|
||||
req_a2.meta["throttler_scopes"] = "slot-a"
|
||||
req_b1 = Request("https://example.org/b1")
|
||||
req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
|
||||
req_b1.meta["throttler_scopes"] = "slot-b"
|
||||
req_c1 = Request("https://example.org/c1")
|
||||
req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
|
||||
req_c1.meta["throttler_scopes"] = "slot-c"
|
||||
|
||||
for request in (req_a1, req_a2, req_b1, req_c1):
|
||||
self.queue.push(request)
|
||||
|
||||
slots = [
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta[Downloader.DOWNLOAD_SLOT],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
self.queue.pop().meta["throttler_scopes"],
|
||||
]
|
||||
|
||||
assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]
|
||||
|
||||
def test_pop_prefers_slot_with_fewer_active_downloads(self):
|
||||
downloader = self.queue._downloader_interface.downloader
|
||||
throttler = self.queue._downloader_interface._throttler
|
||||
assert throttler is not None
|
||||
|
||||
req_a = Request("https://example.org/a")
|
||||
req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
|
||||
req_a.meta["throttler_scopes"] = "slot-a"
|
||||
req_b = Request("https://example.org/b")
|
||||
req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
|
||||
req_b.meta["throttler_scopes"] = "slot-b"
|
||||
req_c = Request("https://example.org/c")
|
||||
req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
|
||||
req_c.meta["throttler_scopes"] = "slot-c"
|
||||
|
||||
for req in (req_a, req_b, req_c):
|
||||
self.queue.push(req)
|
||||
|
||||
downloader.increment("slot-a")
|
||||
downloader.increment("slot-c")
|
||||
throttler._get_scope_manager("slot-a")._active = 1
|
||||
throttler._get_scope_manager("slot-c")._active = 1
|
||||
|
||||
popped = self.queue.pop()
|
||||
assert popped.url == req_b.url
|
||||
|
||||
def test_contains(self):
|
||||
req = Request("https://example.org/")
|
||||
req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"
|
||||
req.meta["throttler_scopes"] = "example-slot"
|
||||
assert "example-slot" not in self.queue
|
||||
self.queue.push(req)
|
||||
assert "example-slot" in self.queue
|
||||
|
|
@ -309,3 +315,263 @@ def test_pop_order(input_, output):
|
|||
actual_output_urls.append(request.url)
|
||||
|
||||
assert actual_output_urls == expected_output_urls
|
||||
|
||||
|
||||
class TestThrottlerAwarePriorityQueue:
|
||||
def _queue(self, crawler, key=""):
|
||||
return build_from_crawler(
|
||||
ThrottlerAwarePriorityQueue,
|
||||
crawler,
|
||||
downstream_queue_cls=FifoMemoryQueue,
|
||||
key=key,
|
||||
)
|
||||
|
||||
async def _push(self, queue, crawler, request):
|
||||
scope_set = frozenset(iter_scopes(await crawler.throttler.get_scopes(request)))
|
||||
queue.push(request, scope_set)
|
||||
|
||||
@coroutine_test
|
||||
async def test_partitions_by_scope_set(self):
|
||||
crawler = get_crawler(Spider)
|
||||
queue = self._queue(crawler)
|
||||
await self._push(queue, crawler, Request("http://a.com/1"))
|
||||
await self._push(queue, crawler, Request("http://a.com/2"))
|
||||
await self._push(queue, crawler, Request("http://b.com/1"))
|
||||
# Two distinct scope sets -> two internal queues, three requests.
|
||||
assert len(queue.pqueues) == 2
|
||||
assert len(queue) == 3
|
||||
|
||||
@coroutine_test
|
||||
async def test_pop_skips_blocked_scope(self):
|
||||
crawler = get_crawler(
|
||||
Spider,
|
||||
settings_dict={
|
||||
"THROTTLER_SCOPES": {"slow.com": {"delay": 1000.0}},
|
||||
"RANDOMIZE_DOWNLOAD_DELAY": False,
|
||||
},
|
||||
)
|
||||
queue = self._queue(crawler)
|
||||
await self._push(queue, crawler, Request("http://slow.com/1"))
|
||||
await self._push(queue, crawler, Request("http://slow.com/2"))
|
||||
await self._push(queue, crawler, Request("http://fast.com/1"))
|
||||
# The first slow request is sendable (no delay accrued yet); after it is
|
||||
# popped (and reserved), the second slow request is blocked, but the
|
||||
# fast one is still served.
|
||||
popped = [queue.pop(), queue.pop(), queue.pop()]
|
||||
urls = [r.url if r else None for r in popped]
|
||||
assert "http://fast.com/1" in urls
|
||||
assert "http://slow.com/1" in urls
|
||||
# The blocked second slow request stays in the queue.
|
||||
assert None in urls
|
||||
assert len(queue) == 1
|
||||
delay = queue.get_next_request_delay()
|
||||
assert delay is not None
|
||||
assert delay == pytest.approx(1000.0, abs=1.0)
|
||||
|
||||
@coroutine_test
|
||||
async def test_pop_holds_request_with_delay(self):
|
||||
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
|
||||
queue = self._queue(crawler)
|
||||
await self._push(
|
||||
queue,
|
||||
crawler,
|
||||
Request("http://slow.com/1", meta={"delay": 1000.0}),
|
||||
)
|
||||
await self._push(queue, crawler, Request("http://fast.com/1"))
|
||||
# The delayed request is held back even though its scope is otherwise
|
||||
# unconstrained; the request without a delay is served.
|
||||
popped = [queue.pop(), queue.pop()]
|
||||
urls = [r.url if r else None for r in popped]
|
||||
assert "http://fast.com/1" in urls
|
||||
assert None in urls
|
||||
assert len(queue) == 1
|
||||
delay = queue.get_next_request_delay()
|
||||
assert delay is not None
|
||||
assert delay == pytest.approx(1000.0, abs=1.0)
|
||||
|
||||
@coroutine_test
|
||||
async def test_delayed_request_does_not_block_scope_set(self):
|
||||
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
|
||||
queue = self._queue(crawler)
|
||||
# Both requests share the same (example.com) scope set; only the first
|
||||
# carries a per-request delay.
|
||||
await self._push(
|
||||
queue,
|
||||
crawler,
|
||||
Request("http://example.com/slow", meta={"delay": 1000.0}),
|
||||
)
|
||||
await self._push(queue, crawler, Request("http://example.com/fast"))
|
||||
# The delayed request is held aside, so the other request in the same
|
||||
# scope set is served right away instead of being stuck behind it.
|
||||
assert queue.pop().url == "http://example.com/fast"
|
||||
# The delayed request is not lost, just not poppable yet.
|
||||
assert queue.pop() is None
|
||||
assert len(queue) == 1
|
||||
assert queue.get_next_request_delay() == pytest.approx(1000.0, abs=1.0)
|
||||
|
||||
@coroutine_test
|
||||
async def test_delayed_request_promoted_when_due(self):
|
||||
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
|
||||
queue = self._queue(crawler)
|
||||
request = Request("http://example.com/slow", meta={"delay": 1000.0})
|
||||
await self._push(queue, crawler, request)
|
||||
assert queue.pop() is None # held back by its per-request delay
|
||||
# Once the delay elapses the request is promoted into its scope-set
|
||||
# queue, served, and flagged so the delay is not applied a second time.
|
||||
queue._promote_ready(queue._delayed[0][0])
|
||||
popped = queue.pop()
|
||||
assert popped is request
|
||||
assert popped.meta["_throttler_delayed"] is True
|
||||
assert len(queue) == 0
|
||||
|
||||
@coroutine_test
|
||||
async def test_delayed_request_persisted_on_close(self):
|
||||
# With a JOBDIR (disk queue), a request held back by its per-request
|
||||
# delay must not be lost on a graceful stop: close() flushes it to disk
|
||||
# so it is restored on resume.
|
||||
crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False})
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
queue = build_from_crawler(
|
||||
ThrottlerAwarePriorityQueue,
|
||||
crawler,
|
||||
downstream_queue_cls=PickleFifoDiskQueue,
|
||||
key=temp_dir,
|
||||
)
|
||||
await self._push(
|
||||
queue,
|
||||
crawler,
|
||||
Request("http://example.com/slow", meta={"delay": 1000.0}),
|
||||
)
|
||||
assert len(queue) == 1 # held in memory, not yet in any scope-set queue
|
||||
state = queue.close() # graceful stop
|
||||
|
||||
resumed = build_from_crawler(
|
||||
ThrottlerAwarePriorityQueue,
|
||||
crawler,
|
||||
downstream_queue_cls=PickleFifoDiskQueue,
|
||||
key=temp_dir,
|
||||
startprios=state,
|
||||
)
|
||||
assert len(resumed) == 1
|
||||
popped = resumed.pop()
|
||||
assert popped is not None
|
||||
assert popped.url == "http://example.com/slow"
|
||||
# Its delay is marked consumed, so it does not re-block on resume.
|
||||
assert popped.meta["_throttler_delayed"] is True
|
||||
resumed.close()
|
||||
|
||||
@coroutine_test
|
||||
async def test_least_loaded_first(self):
|
||||
crawler = get_crawler(
|
||||
Spider,
|
||||
settings_dict={
|
||||
"THROTTLER_SCOPES": {
|
||||
"a.com": {"concurrency": 4},
|
||||
"b.com": {"concurrency": 4},
|
||||
}
|
||||
},
|
||||
)
|
||||
queue = self._queue(crawler)
|
||||
# Make a.com busier than b.com.
|
||||
busy = Request("http://a.com/0")
|
||||
await crawler.throttler.get_scopes(busy)
|
||||
crawler.throttler.reserve(busy)
|
||||
await self._push(queue, crawler, Request("http://a.com/1"))
|
||||
await self._push(queue, crawler, Request("http://b.com/1"))
|
||||
# Equal priority, so the lower-load scope (b.com) is served first.
|
||||
assert queue.pop().url == "http://b.com/1"
|
||||
|
||||
@coroutine_test
|
||||
async def test_priority_beats_load(self):
|
||||
crawler = get_crawler(
|
||||
Spider,
|
||||
settings_dict={
|
||||
"THROTTLER_SCOPES": {
|
||||
"a.com": {"concurrency": 4},
|
||||
"b.com": {"concurrency": 4},
|
||||
}
|
||||
},
|
||||
)
|
||||
queue = self._queue(crawler)
|
||||
# Make a.com busier than b.com.
|
||||
busy = Request("http://a.com/0")
|
||||
await crawler.throttler.get_scopes(busy)
|
||||
crawler.throttler.reserve(busy)
|
||||
# The a.com request has higher priority, and a.com still has room, so it
|
||||
# is served first despite a.com being the busier scope.
|
||||
await self._push(queue, crawler, Request("http://a.com/1", priority=10))
|
||||
await self._push(queue, crawler, Request("http://b.com/1"))
|
||||
assert queue.pop().url == "http://a.com/1"
|
||||
|
||||
@coroutine_test
|
||||
async def test_empty_and_close(self):
|
||||
crawler = get_crawler(Spider)
|
||||
queue = self._queue(crawler)
|
||||
assert queue.pop() is None
|
||||
assert queue.get_next_request_delay() is None
|
||||
await self._push(queue, crawler, Request("http://a.com/1"))
|
||||
assert queue.close() != {}
|
||||
|
||||
@coroutine_test
|
||||
async def test_get_next_request_delay_zero_when_ready(self):
|
||||
crawler = get_crawler(Spider)
|
||||
queue = self._queue(crawler)
|
||||
await self._push(queue, crawler, Request("http://a.com/1"))
|
||||
# A sendable head means no wait is needed.
|
||||
assert queue.get_next_request_delay() == 0.0
|
||||
|
||||
@coroutine_test
|
||||
async def test_get_next_request_delay_ignores_empty_queues(self):
|
||||
crawler = get_crawler(Spider)
|
||||
queue = self._queue(crawler)
|
||||
# An empty (but still registered) internal queue is skipped.
|
||||
queue.pqueues[frozenset({"a.com"})] = queue._pqfactory(frozenset({"a.com"}))
|
||||
assert queue.get_next_request_delay() is None
|
||||
|
||||
@coroutine_test
|
||||
async def test_get_next_request_delay_keeps_minimum(self):
|
||||
crawler = get_crawler(
|
||||
Spider,
|
||||
settings_dict={
|
||||
"THROTTLER_SCOPES": {
|
||||
"a.com": {"delay": 10.0},
|
||||
"b.com": {"delay": 1000.0},
|
||||
},
|
||||
"RANDOMIZE_DOWNLOAD_DELAY": False,
|
||||
},
|
||||
)
|
||||
queue = self._queue(crawler)
|
||||
# Two requests per scope so a blocked head remains after the first one
|
||||
# (sendable, since no delay has accrued yet) is popped and reserved.
|
||||
await self._push(queue, crawler, Request("http://a.com/1"))
|
||||
await self._push(queue, crawler, Request("http://a.com/2"))
|
||||
await self._push(queue, crawler, Request("http://b.com/1"))
|
||||
await self._push(queue, crawler, Request("http://b.com/2"))
|
||||
queue.pop()
|
||||
queue.pop()
|
||||
# Both scopes are now time-blocked; the smaller per-scope delay wins,
|
||||
# so the larger one exercises the "not below the running minimum" branch.
|
||||
delay = queue.get_next_request_delay()
|
||||
assert delay == pytest.approx(10.0, abs=1.0)
|
||||
|
||||
@coroutine_test
|
||||
async def test_pop_handles_drained_selected_queue(self):
|
||||
crawler = get_crawler(Spider)
|
||||
queue = self._queue(crawler)
|
||||
await self._push(queue, crawler, Request("http://a.com/1"))
|
||||
inner = next(iter(queue.pqueues.values()))
|
||||
# _select() still reports a sendable head, but pop() yields nothing: the
|
||||
# request-is-None guard must not try to reserve a missing request.
|
||||
inner.pop = lambda: None
|
||||
assert queue.pop() is None
|
||||
|
||||
def test_non_dict_slot_startprios(self):
|
||||
crawler = get_crawler(Spider)
|
||||
with pytest.raises(ValueError, match="slot_startprios"):
|
||||
build_from_crawler(
|
||||
ThrottlerAwarePriorityQueue,
|
||||
crawler,
|
||||
downstream_queue_cls=FifoMemoryQueue,
|
||||
key="",
|
||||
startprios=[1, 2, 3],
|
||||
)
|
||||
|
|
|
|||
|
|
@ -88,6 +88,16 @@ class BaseRobotParserTest:
|
|||
assert rp.allowed("https://site.local/index.html", "*")
|
||||
assert rp.allowed("https://site.local/disallowed", "*")
|
||||
|
||||
def test_crawl_delay(self):
|
||||
robotstxt_body = b"User-agent: *\nCrawl-delay: 10\n"
|
||||
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
|
||||
assert rp.crawl_delay("*") == 10.0
|
||||
|
||||
def test_crawl_delay_unset(self):
|
||||
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
|
||||
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
|
||||
assert rp.crawl_delay("*") is None
|
||||
|
||||
def test_unicode_url_and_useragent(self):
|
||||
robotstxt_robotstxt_body = """
|
||||
User-Agent: *
|
||||
|
|
@ -171,6 +181,9 @@ class TestRerpRobotParser(BaseRobotParserTest):
|
|||
def test_length_based_precedence(self):
|
||||
pytest.skip("Rerp does not support length based directives precedence.")
|
||||
|
||||
def test_crawl_delay(self):
|
||||
pytest.skip("Rerp does not expose Crawl-delay directives.")
|
||||
|
||||
|
||||
class TestProtegoRobotParser(BaseRobotParserTest):
|
||||
def setup_method(self):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import deque
|
||||
|
|
@ -10,7 +11,7 @@ from unittest.mock import Mock
|
|||
import pytest
|
||||
|
||||
from scrapy.core.downloader import Downloader
|
||||
from scrapy.core.scheduler import BaseScheduler, Scheduler
|
||||
from scrapy.core.scheduler import BaseScheduler, Scheduler, ThrottlerAwareScheduler
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request
|
||||
|
|
@ -26,6 +27,8 @@ if TYPE_CHECKING:
|
|||
from collections.abc import AsyncGenerator
|
||||
from pathlib import Path
|
||||
|
||||
from scrapy.http.request import CallbackT
|
||||
|
||||
|
||||
class MemoryScheduler(BaseScheduler):
|
||||
paused = False
|
||||
|
|
@ -406,3 +409,251 @@ class TestIncompatibility:
|
|||
ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP"
|
||||
):
|
||||
self._incompatible()
|
||||
|
||||
|
||||
_THROTTLER_AWARE_PQ = "scrapy.pqueues.ThrottlerAwarePriorityQueue"
|
||||
|
||||
|
||||
class TestThrottlerAwareScheduler:
|
||||
def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler:
|
||||
settings = {
|
||||
"SCHEDULER_PRIORITY_QUEUE": _THROTTLER_AWARE_PQ,
|
||||
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
|
||||
**(settings_dict or {}),
|
||||
}
|
||||
return get_crawler(Spider, settings)
|
||||
|
||||
def _scheduler(self, crawler: Crawler) -> ThrottlerAwareScheduler:
|
||||
spider = Spider(name="spider")
|
||||
crawler.spider = spider
|
||||
scheduler = ThrottlerAwareScheduler.from_crawler(crawler)
|
||||
scheduler.open(spider)
|
||||
return scheduler
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_async_and_dequeue(self) -> None:
|
||||
scheduler = self._scheduler(self._crawler())
|
||||
assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True
|
||||
assert scheduler.has_pending_requests()
|
||||
assert len(scheduler) == 1
|
||||
request = scheduler.next_request()
|
||||
assert request is not None
|
||||
assert request.url == "http://a.com/1"
|
||||
assert scheduler.next_request() is None
|
||||
assert not scheduler.has_pending_requests()
|
||||
scheduler.close("finished")
|
||||
|
||||
def test_sync_enqueue_raises(self) -> None:
|
||||
scheduler = self._scheduler(self._crawler())
|
||||
with pytest.raises(RuntimeError, match="asynchronous enqueue path"):
|
||||
scheduler.enqueue_request(Request("http://a.com/1"))
|
||||
scheduler.close("finished")
|
||||
|
||||
def test_requires_throttler_aware_priority_queue(self) -> None:
|
||||
crawler = self._crawler(
|
||||
{"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ScrapyPriorityQueue"}
|
||||
)
|
||||
spider = Spider(name="spider")
|
||||
crawler.spider = spider
|
||||
scheduler = ThrottlerAwareScheduler.from_crawler(crawler)
|
||||
with pytest.raises(ValueError, match="throttler-aware priority queue"):
|
||||
scheduler.open(spider)
|
||||
|
||||
@coroutine_test
|
||||
async def test_delay_blocks_and_reports_delay(self) -> None:
|
||||
crawler = self._crawler(
|
||||
{
|
||||
"THROTTLER_SCOPES": {"slow.com": {"delay": 1000.0}},
|
||||
"RANDOMIZE_DOWNLOAD_DELAY": False,
|
||||
},
|
||||
)
|
||||
scheduler = self._scheduler(crawler)
|
||||
await scheduler.enqueue_request_async(Request("http://slow.com/1"))
|
||||
await scheduler.enqueue_request_async(Request("http://slow.com/2"))
|
||||
# The first request is sendable; the second is blocked by the delay.
|
||||
first = scheduler.next_request()
|
||||
assert first is not None
|
||||
assert scheduler.next_request() is None
|
||||
assert scheduler.has_pending_requests()
|
||||
assert scheduler.get_next_request_delay() == pytest.approx(1000.0, abs=1.0)
|
||||
scheduler.close("finished")
|
||||
|
||||
@coroutine_test
|
||||
async def test_no_delay_when_only_concurrency_blocked(self) -> None:
|
||||
crawler = self._crawler(
|
||||
{"THROTTLER_SCOPES": {"slow.com": {"concurrency": 1}}},
|
||||
)
|
||||
scheduler = self._scheduler(crawler)
|
||||
await scheduler.enqueue_request_async(Request("http://slow.com/1"))
|
||||
await scheduler.enqueue_request_async(Request("http://slow.com/2"))
|
||||
assert scheduler.next_request() is not None
|
||||
assert scheduler.next_request() is None
|
||||
# A purely concurrency-blocked state has no time-based wakeup.
|
||||
assert scheduler.get_next_request_delay() is None
|
||||
scheduler.close("finished")
|
||||
|
||||
@coroutine_test
|
||||
async def test_delayed_request_survives_jobdir_stop(self, tmp_path: Path) -> None:
|
||||
# A request held back by its per-request delay must not be
|
||||
# lost on a graceful stop when a JOBDIR is configured: it is flushed to
|
||||
# the disk queue on close and restored on resume.
|
||||
crawler = self._crawler(
|
||||
{"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False}
|
||||
)
|
||||
scheduler = self._scheduler(crawler)
|
||||
request = Request("http://a.com/slow", meta={"delay": 1000.0})
|
||||
assert await scheduler.enqueue_request_async(request) is True
|
||||
assert len(scheduler) == 1
|
||||
# The delay holds it back, so nothing is dequeued before the stop.
|
||||
assert scheduler.next_request() is None
|
||||
scheduler.close("finished")
|
||||
|
||||
# Resume from the same JOBDIR: the request is still there and, having
|
||||
# been held once, is now sendable.
|
||||
resumed = self._scheduler(
|
||||
self._crawler({"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False})
|
||||
)
|
||||
assert len(resumed) == 1
|
||||
resumed_request = resumed.next_request()
|
||||
assert resumed_request is not None
|
||||
assert resumed_request.url == "http://a.com/slow"
|
||||
resumed.close("finished")
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_async_filters_duplicates(self) -> None:
|
||||
crawler = self._crawler(
|
||||
{"DUPEFILTER_CLASS": "scrapy.dupefilters.RFPDupeFilter"}
|
||||
)
|
||||
scheduler = self._scheduler(crawler)
|
||||
assert crawler.spider is not None
|
||||
crawler.spider.crawler = crawler # the dupefilter logs via spider.crawler
|
||||
assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True
|
||||
# The same request is filtered out the second time around.
|
||||
assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is False
|
||||
assert len(scheduler) == 1
|
||||
scheduler.close("finished")
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_async_unserializable_falls_back_to_memory(
|
||||
self, tmp_path: Path, caplog
|
||||
) -> None:
|
||||
crawler = self._crawler({"JOBDIR": str(tmp_path), "SCHEDULER_DEBUG": True})
|
||||
scheduler = self._scheduler(crawler)
|
||||
# A lambda callback cannot be serialized to disk, so the request falls
|
||||
# back to the in-memory queue and the failure is logged once.
|
||||
request = Request(
|
||||
"http://a.com/1", callback=cast("CallbackT", lambda response: None)
|
||||
)
|
||||
with caplog.at_level(logging.WARNING, logger="scrapy.core.scheduler"):
|
||||
assert await scheduler.enqueue_request_async(request) is True
|
||||
assert "Unable to serialize request" in caplog.text
|
||||
assert crawler.stats is not None
|
||||
assert crawler.stats.get_value("scheduler/unserializable") == 1
|
||||
assert crawler.stats.get_value("scheduler/enqueued/memory") == 1
|
||||
scheduler.close("finished")
|
||||
|
||||
@coroutine_test
|
||||
async def test_enqueue_async_unserializable_without_debug(
|
||||
self, tmp_path: Path
|
||||
) -> None:
|
||||
# Same fallback as above, but with SCHEDULER_DEBUG off the failure is
|
||||
# tracked in stats without logging a warning.
|
||||
crawler = self._crawler({"JOBDIR": str(tmp_path)})
|
||||
scheduler = self._scheduler(crawler)
|
||||
request = Request(
|
||||
"http://a.com/1", callback=cast("CallbackT", lambda response: None)
|
||||
)
|
||||
assert await scheduler.enqueue_request_async(request) is True
|
||||
assert crawler.stats is not None
|
||||
assert crawler.stats.get_value("scheduler/unserializable") == 1
|
||||
scheduler.close("finished")
|
||||
|
||||
@coroutine_test
|
||||
async def test_resume_from_disk(self, tmp_path: Path) -> None:
|
||||
settings = {"JOBDIR": str(tmp_path)}
|
||||
scheduler = self._scheduler(self._crawler(settings))
|
||||
await scheduler.enqueue_request_async(Request("http://a.com/1"))
|
||||
await scheduler.enqueue_request_async(Request("http://b.com/1"))
|
||||
scheduler.close("shutdown")
|
||||
|
||||
scheduler2 = self._scheduler(self._crawler(settings))
|
||||
assert len(scheduler2) == 2
|
||||
urls = set()
|
||||
while (request := scheduler2.next_request()) is not None:
|
||||
urls.add(request.url)
|
||||
assert urls == {"http://a.com/1", "http://b.com/1"}
|
||||
scheduler2.close("finished")
|
||||
|
||||
|
||||
class TestIntegrationWithThrottlerAwareScheduler:
|
||||
@inline_callbacks_test
|
||||
def test_integration(self):
|
||||
crawler = get_crawler(
|
||||
spidercls=StartUrlsSpider,
|
||||
settings_dict={
|
||||
"SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler",
|
||||
"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue",
|
||||
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
|
||||
},
|
||||
)
|
||||
with MockServer() as mockserver:
|
||||
url = mockserver.url("/status?n=200", is_secure=False)
|
||||
start_urls = [url] * 6
|
||||
yield crawler.crawl(start_urls)
|
||||
assert crawler.stats.get_value("downloader/response_count") == len(
|
||||
start_urls
|
||||
)
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_integration_follow_requests(self):
|
||||
# Exercises the engine's asynchronous enqueue path for requests yielded
|
||||
# from a callback (not just start requests), and the idle guard that
|
||||
# keeps the spider open while an enqueue is in flight.
|
||||
class FollowSpider(Spider):
|
||||
name = "follow"
|
||||
|
||||
def __init__(self, base_url, **kwargs):
|
||||
self.base_url = base_url
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def start(self):
|
||||
yield Request(self.base_url + "/status?n=200")
|
||||
|
||||
def parse(self, response):
|
||||
if b"follow" not in response.url.encode():
|
||||
yield Request(response.url + "&follow=1", callback=self.parse)
|
||||
|
||||
with MockServer() as mockserver:
|
||||
base_url = mockserver.url("", is_secure=False).rstrip("/")
|
||||
crawler = get_crawler(
|
||||
spidercls=FollowSpider,
|
||||
settings_dict={
|
||||
"SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler",
|
||||
"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue",
|
||||
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
|
||||
},
|
||||
)
|
||||
yield crawler.crawl(base_url=base_url)
|
||||
assert crawler.stats.get_value("downloader/response_count") == 2
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_integration_with_delay(self):
|
||||
# A small per-scope delay forces the engine to arm the throttler wakeup
|
||||
# timer between requests; the crawl must still complete.
|
||||
crawler = get_crawler(
|
||||
spidercls=StartUrlsSpider,
|
||||
settings_dict={
|
||||
"SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler",
|
||||
"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue",
|
||||
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
|
||||
"RANDOMIZE_DOWNLOAD_DELAY": False,
|
||||
"THROTTLER_SCOPES": {"127.0.0.1": {"delay": 0.05}},
|
||||
},
|
||||
)
|
||||
with MockServer() as mockserver:
|
||||
url = mockserver.url("/status?n=200", is_secure=False)
|
||||
start_urls = [url] * 4
|
||||
yield crawler.crawl(start_urls)
|
||||
assert crawler.stats.get_value("downloader/response_count") == len(
|
||||
start_urls
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -13,6 +13,8 @@ from scrapy.utils.asyncio import (
|
|||
AsyncioLoopingCall,
|
||||
_parallel_asyncio,
|
||||
is_asyncio_available,
|
||||
sleep,
|
||||
wait_for_first,
|
||||
)
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
|
|
@ -158,3 +160,42 @@ class TestAsyncioLoopingCall:
|
|||
looping_call.start(0.1)
|
||||
assert not looping_call.running
|
||||
assert "Error calling the AsyncioLoopingCall function" in caplog.text
|
||||
|
||||
|
||||
class TestSleep:
|
||||
@coroutine_test
|
||||
async def test_sleep(self):
|
||||
# A zero-second sleep completes without error under any reactor mode.
|
||||
await sleep(0)
|
||||
|
||||
|
||||
class TestWaitForFirst:
|
||||
@coroutine_test
|
||||
async def test_empty(self):
|
||||
# No deferreds -> two empty sets, returned immediately.
|
||||
assert await wait_for_first([]) == (set(), set())
|
||||
|
||||
@coroutine_test
|
||||
async def test_returns_done_deferred(self):
|
||||
fired: Deferred[None] = Deferred()
|
||||
fired.callback(None)
|
||||
done, pending = await wait_for_first([fired], timeout=1)
|
||||
assert done == {fired}
|
||||
assert pending == set()
|
||||
|
||||
@coroutine_test
|
||||
async def test_no_timeout(self):
|
||||
# timeout=None -> no timeout deferred is created.
|
||||
fired: Deferred[None] = Deferred()
|
||||
fired.callback(None)
|
||||
done, pending = await wait_for_first([fired])
|
||||
assert done == {fired}
|
||||
assert pending == set()
|
||||
|
||||
@coroutine_test
|
||||
async def test_timeout_elapses(self):
|
||||
# When the timeout fires first, every input deferred is still pending.
|
||||
never: Deferred[None] = Deferred()
|
||||
done, pending = await wait_for_first([never], timeout=0.01)
|
||||
assert done == set()
|
||||
assert pending == {never}
|
||||
|
|
|
|||
4
tox.ini
4
tox.ini
|
|
@ -144,7 +144,7 @@ deps =
|
|||
lxml==4.6.4
|
||||
parsel==1.5.0
|
||||
pyOpenSSL==22.0.0
|
||||
queuelib==1.4.2
|
||||
queuelib==1.6.1
|
||||
service_identity==23.1.0
|
||||
w3lib==1.17.0
|
||||
zope.interface==5.1.0
|
||||
|
|
@ -260,7 +260,7 @@ deps =
|
|||
lxml==5.3.2
|
||||
parsel==1.5.0
|
||||
pyOpenSSL==24.3.0
|
||||
queuelib==1.4.2
|
||||
queuelib==1.6.1
|
||||
service_identity==23.1.0
|
||||
# w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
|
||||
# an inline flag placement that Python 3.11 treats as an error: global
|
||||
|
|
|
|||
Loading…
Reference in New Issue