diff --git a/docs/conf.py b/docs/conf.py index b950c4ee2..7770e550b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -31,6 +31,7 @@ extensions = [ "sphinx_scrapy", "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", + "sphinx_reredirects", "sphinx_rtd_dark_mode", ] @@ -141,12 +142,21 @@ coverage_ignore_pyobjects = [ r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] +# -- Options for the autodoc extension ---------------------------------------- +autodoc_type_aliases = { + "RequestScopes": "RequestScopes", +} # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration intersphinx_disabled_reftypes: Sequence[str] = [] +# -- sphinx-reredirects ------------------------------------------------------- +redirects = { + "topics/autothrottle": "throttling.html", +} + # sphinx-scrapy --------------------------------------------------------------- scrapy_intersphinx_enable = [ diff --git a/docs/index.rst b/docs/index.rst index 8e8624a22..ac826793f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,7 +158,7 @@ Solving specific problems topics/leaks topics/media-pipeline topics/deploy - topics/autothrottle + topics/throttling topics/benchmarking topics/jobs topics/coroutines @@ -198,8 +198,9 @@ Solving specific problems :doc:`topics/deploy` Deploying your Scrapy spiders and run them in a remote server. -:doc:`topics/autothrottle` - Adjust crawl rate dynamically based on load. +:doc:`topics/throttling` + Control request throttling to avoid overloading websites and comply with + rate limits. :doc:`topics/benchmarking` Check how Scrapy performs on your hardware. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index ee91ce7ca..c5ff76246 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -81,11 +81,9 @@ error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you -control over the politeness of the crawl through :ref:`a few settings -`. You can do things like setting a download delay between -each request, limiting the amount of concurrent requests per domain or per IP, and -even :ref:`using an auto-throttling extension ` that tries -to figure these settings out automatically. +control over :ref:`throttling `, e.g. you can set a delay between +requests to the same domain, set a maximum concurrency per domain, or customize +the :ref:`backoff ` behavior. .. note:: diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..cb9118304 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -130,6 +130,28 @@ and defines some attributes and methods: the scraped data as dicts and also finding new URLs to follow and creating new requests (:class:`~scrapy.Request`) from them. +Crawling faster for this tutorial +--------------------------------- + +By default, Scrapy :ref:`throttles ` crawls to be polite: it sends +at most one request at a time to each website, waiting one second between +requests. That keeps you from overwhelming websites, but it also makes crawling +slow, and it means the requests above are sent one after another rather than in +parallel. + +``quotes.toscrape.com`` is a sandbox meant for scraping practice, so we can +safely crawl it faster. Open ``tutorial/settings.py`` and add a +:setting:`THROTTLER_SCOPES` entry that raises the concurrency and lowers the +delay for that domain only: + +.. code-block:: python + + THROTTLER_SCOPES = { + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, + } + +The polite defaults still apply to every other website. + How to run our spider --------------------- diff --git a/docs/news.rst b/docs/news.rst index b8b976df9..7a87e31b3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1173,7 +1173,7 @@ Deprecations (:issue:`7005`, :issue:`7043`) - The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + ``CONCURRENT_REQUESTS_PER_DOMAIN`` instead. (:issue:`6917`, :issue:`6921`) - The ``scrapy.core.downloader.handlers.http`` module is deprecated. You @@ -1505,7 +1505,7 @@ Scrapy 2.13.3 (2025-07-02) -------------------------- - Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the + ``CONCURRENT_REQUESTS_PER_DOMAIN`` (from ``8`` to ``1``) in the default project template. (:issue:`6597`, :issue:`6918`, :issue:`6923`) @@ -2491,10 +2491,9 @@ New features :meth:`~scrapy.crawler.Crawler.get_spider_middleware`. (:issue:`6181`) -- Slot delay updates by the :ref:`AutoThrottle extension - ` based on response latencies can now be disabled for - specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta - key. +- Slot delay updates by the ``scrapy.extensions.throttle.AutoThrottle`` + extension based on response latencies can now be disabled for specific + requests via the ``autothrottle_dont_adjust_delay`` meta key. (:issue:`6246`, :issue:`6527`) - If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, @@ -3260,9 +3259,9 @@ New features ~~~~~~~~~~~~ - Settings corresponding to :setting:`DOWNLOAD_DELAY`, - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + ``CONCURRENT_REQUESTS_PER_DOMAIN`` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis - via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`) - Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) @@ -6276,7 +6275,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -6476,8 +6475,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * From :class:`~scrapy.spiders.Spider` (and subclasses): - * ``DOWNLOAD_DELAY`` (use :ref:`download_delay - `) + * ``DOWNLOAD_DELAY`` (use ``download_delay``) * ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`) @@ -7450,7 +7448,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Item loaders now support nested loaders (:issue:`1467`). - ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`). - - Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved + - Added setting ``AUTOTHROTTLE_TARGET_CONCURRENCY`` and improved AutoThrottle docs (:issue:`1324`). - Added ``response.text`` to get body as unicode (:issue:`1730`). - Anonymous S3 connections (:issue:`1358`). @@ -8653,7 +8651,7 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command -- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` +- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with ``AUTOTHROTTLE_ENABLED`` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. @@ -8755,7 +8753,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, ``CONCURRENT_REQUESTS_PER_DOMAIN``, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/docs/requirements.in b/docs/requirements.in index a1f3a7468..a964d6498 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -3,6 +3,7 @@ pydantic scrapy-spider-metadata sphinx sphinx-notfound-page +sphinx-reredirects sphinx-rtd-theme sphinx-rtd-dark-mode sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index a5cbad302..655b45e72 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -134,6 +134,7 @@ sphinx==9.1.0 # sphinx-llms-txt # sphinx-markdown-builder # sphinx-notfound-page + # sphinx-reredirects # sphinx-rtd-theme # sphinx-scrapy # sphinxcontrib-jquery @@ -147,6 +148,8 @@ sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builde # via sphinx-scrapy sphinx-notfound-page==1.1.0 # via -r docs/requirements.in +sphinx-reredirects==1.1.0 + # via -r docs/requirements.in sphinx-rtd-dark-mode==1.3.0 # via -r docs/requirements.in sphinx-rtd-theme==3.1.0 diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 19082d9d7..edfc37959 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -93,6 +93,8 @@ how you :ref:`configure the downloader middlewares modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. + .. autoattribute:: throttler + .. attribute:: spider Spider currently being crawled. This is an instance of the spider class diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst deleted file mode 100644 index d0321c906..000000000 --- a/docs/topics/autothrottle.rst +++ /dev/null @@ -1,190 +0,0 @@ -.. _topics-autothrottle: - -====================== -AutoThrottle extension -====================== - -This is an extension for automatically throttling crawling speed based on load -of both the Scrapy server and the website you are crawling. - -Design goals -============ - -1. be nicer to sites instead of using default download delay of zero -2. automatically adjust Scrapy to the optimum crawling speed, so the user - doesn't have to tune the download delays to find the optimum one. - The user only needs to specify the maximum concurrent requests - it allows, and the extension does the rest. - -.. _autothrottle-algorithm: - -How it works -============ - -Scrapy allows defining the concurrency and delay of different download slots, -e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are -assigned to slots based on their URL domain, although it is possible to -customize the download slot of any request. - -The AutoThrottle extension adjusts the delay of each download slot dynamically, -to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent -requests on average to each remote website. - -It uses download latency to compute the delays. The main idea is the -following: if a server needs ``latency`` seconds to respond, a client -should send a request each ``latency/N`` seconds to have ``N`` requests -processed in parallel. - -Instead of adjusting the delays one can just set a small fixed -download delay and impose hard limits on concurrency using -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar -effect, but there are some important differences: - -* because the download delay is small there will be occasional bursts - of requests; -* often non-200 (error) responses can be returned faster than regular - responses, so with a small download delay and a hard concurrency limit - crawler will be sending requests to server faster when server starts to - return errors. But this is an opposite of what crawler should do - in case - of errors it makes more sense to slow down: these errors may be caused by - the high request rate. - -AutoThrottle doesn't have these issues. - -Throttling algorithm -==================== - -AutoThrottle algorithm adjusts download delays based on the following rules: - -1. spiders always start with a download delay of - :setting:`AUTOTHROTTLE_START_DELAY`; -2. when a response is received, the target download delay is calculated as - ``latency / N`` where ``latency`` is a latency of the response, - and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`. -3. download delay for next requests is set to the average of previous - download delay and the target download delay; -4. latencies of non-200 responses are not allowed to decrease the delay; -5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater - than :setting:`AUTOTHROTTLE_MAX_DELAY` - -.. note:: The AutoThrottle extension honours the standard Scrapy settings for - concurrency and delay. This means that it will respect - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - never set a download delay lower than :setting:`DOWNLOAD_DELAY`. - -.. _download-latency: - -In Scrapy, the download latency is measured as the time elapsed between -establishing the TCP connection and receiving the HTTP headers. - -Note that these latencies are very hard to measure accurately in a cooperative -multitasking environment because Scrapy may be busy processing a spider -callback, for example, and unable to attend downloads. However, these latencies -should still give a reasonable estimate of how busy Scrapy (and ultimately, the -server) is, and this extension builds on that premise. - -.. reqmeta:: autothrottle_dont_adjust_delay - -Prevent specific requests from triggering slot delay adjustments -================================================================ - -AutoThrottle adjusts the delay of download slots based on the latencies of -responses that belong to that download slot. The only exceptions are non-200 -responses, which are only taken into account to increase that delay, but -ignored if they would decrease that delay. - -You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to -``True`` in any request to prevent its response latency from impacting the -delay of its download slot: - -.. code-block:: python - - from scrapy import Request - - Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) - -Note, however, that AutoThrottle still determines the starting delay of every -download slot by setting the ``download_delay`` attribute on the running -spider. If you want AutoThrottle not to impact a download slot at all, in -addition to setting this meta key in all requests that use that download slot, -you might want to set a custom value for the ``delay`` attribute of that -download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. - -Settings -======== - -The settings used to control the AutoThrottle extension are: - -* :setting:`AUTOTHROTTLE_ENABLED` -* :setting:`AUTOTHROTTLE_START_DELAY` -* :setting:`AUTOTHROTTLE_MAX_DELAY` -* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` -* :setting:`AUTOTHROTTLE_DEBUG` -* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -* :setting:`DOWNLOAD_DELAY` - -For more information see :ref:`autothrottle-algorithm`. - -.. setting:: AUTOTHROTTLE_ENABLED - -AUTOTHROTTLE_ENABLED -~~~~~~~~~~~~~~~~~~~~ - -Default: ``False`` - -Enables the AutoThrottle extension. - -.. setting:: AUTOTHROTTLE_START_DELAY - -AUTOTHROTTLE_START_DELAY -~~~~~~~~~~~~~~~~~~~~~~~~ - -Default: ``5.0`` - -The initial download delay (in seconds). - -.. setting:: AUTOTHROTTLE_MAX_DELAY - -AUTOTHROTTLE_MAX_DELAY -~~~~~~~~~~~~~~~~~~~~~~ - -Default: ``60.0`` - -The maximum download delay (in seconds) to be set in case of high latencies. - -.. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY - -AUTOTHROTTLE_TARGET_CONCURRENCY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Default: ``1.0`` - -Average number of requests Scrapy should be sending in parallel to remote -websites. It must be higher than ``0.0``. - -By default, AutoThrottle adjusts the delay to send a single -concurrent request to each of the remote websites. Set this option to -a higher value (e.g. ``2.0``) to increase the throughput and the load on remote -servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value -(e.g. ``0.5``) makes the crawler more conservative and polite. - -Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected -when AutoThrottle extension is enabled. This means that if -``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number -of concurrent requests. - -At every given time point Scrapy can be sending more or less concurrent -requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested -value the crawler tries to approach, not a hard limit. - -.. setting:: AUTOTHROTTLE_DEBUG - -AUTOTHROTTLE_DEBUG -~~~~~~~~~~~~~~~~~~ - -Default: ``False`` - -Enable AutoThrottle debug mode which will display stats on every response -received, so you can see how the throttling parameters are being adjusted in -real time. diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index cace1f883..ffcff60ae 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -47,8 +47,8 @@ Increase concurrency ==================== Concurrency is the number of requests that are processed in parallel. There is -a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). +a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional per-scope +(per-domain by default) limit (:setting:`THROTTLER_SCOPE_CONCURRENCY`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/components.rst b/docs/topics/components.rst index c0df86922..8f9b3865a 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,37 +9,24 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`ADDONS` - -- :setting:`TWISTED_DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SCHEDULER_START_DISK_QUEUE` - -- :setting:`SCHEDULER_START_MEMORY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`THROTTLER` +- :setting:`THROTTLER_SCOPE_MANAGER` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 5649453b1..fa353d126 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -169,6 +169,13 @@ middleware, see the :ref:`downloader middleware usage guide For a list of the components enabled by default (and their orders) see the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting. +.. _backoff-mw: + +BackoffMiddleware +----------------- + +.. autoclass:: scrapy.downloadermiddlewares.backoff.BackoffMiddleware + .. _cookies-mw: CookiesMiddleware @@ -942,6 +949,22 @@ Default: ``20`` The maximum number of redirections that will be followed for a single request. If maximum redirections are exceeded, the request is aborted and ignored. +.. setting:: REDIRECT_MAX_DELAY + +REDIRECT_MAX_DELAY +^^^^^^^^^^^^^^^^^^ + +Default: ``120.0`` + +If a redirect response provides a ``Retry-After`` header, the redirect is only +followed after the specified delay. This setting caps that delay to a specific +number of seconds. Set to ``0.0`` to disable these delays altogether. + +Note that redirect delays only affect the request being redirected, they do not +delay other requests to the same domain. ``Retry-After`` only does that during +:ref:`backoff `. + + MetaRefreshMiddleware --------------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f2fcfb0b5..a9fd6fd87 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -706,7 +706,6 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: * :reqmeta:`allow_offsite` -* :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` @@ -714,6 +713,7 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` +* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` @@ -733,6 +733,8 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`delay` +* :reqmeta:`throttler_scopes` * :reqmeta:`verbatim_url` .. reqmeta:: bindaddress diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 11251a92c..6d057626a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -538,21 +538,9 @@ CONCURRENT_REQUESTS Default: ``16`` -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +Maximum number of total concurrent requests allowed. -.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - -CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------- - -Default: ``1`` (:ref:`fallback `: ``8``) - -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed to any single domain. - -See also: :ref:`topics-autothrottle` and its -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. +.. seealso:: :ref:`throttling` .. setting:: DEFAULT_DROPITEM_LOG_LEVEL @@ -873,6 +861,7 @@ Default: "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, + "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 650, "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, @@ -894,45 +883,6 @@ Default: ``True`` Whether to enable downloader stats collection. -.. setting:: DOWNLOAD_DELAY - -DOWNLOAD_DELAY --------------- - -Default: ``1`` (:ref:`fallback `: ``0``) - -Minimum seconds to wait between 2 consecutive requests to the same domain. - -Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting -servers too hard. - -Decimal numbers are supported. For example, to send a maximum of 4 requests -every 10 seconds:: - - DOWNLOAD_DELAY = 2.5 - -This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting, which is enabled by default. - -Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain -concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response -time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective -concurrency for that domain is 1. When testing throttling configurations, it -usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first, -and only increase :setting:`DOWNLOAD_DELAY` once -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is -desired. - -.. _spider-download_delay-attribute: - -.. note:: - - This delay can be set per spider using :attr:`download_delay` spider attribute. - -It is also possible to change this setting per domain, although it requires -non-trivial code. See the implementation of the :ref:`AutoThrottle -` extension for an example. - .. setting:: DOWNLOAD_BIND_ADDRESS DOWNLOAD_BIND_ADDRESS @@ -1039,30 +989,6 @@ handler (without replacement), place this in your ``settings.py``: :ref:`security-local-resources` -.. setting:: DOWNLOAD_SLOTS - -DOWNLOAD_SLOTS --------------- - -Default: ``{}`` - -Allows to define concurrency/delay parameters on per slot (domain) basis: - - .. code-block:: python - - DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, - "books.toscrape.com": {"delay": 3, "randomize_delay": False}, - } - -.. note:: - - For other downloader slots default settings values will be used: - - - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` - .. setting:: DOWNLOAD_TIMEOUT @@ -1733,26 +1659,6 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' -.. setting:: RANDOMIZE_DOWNLOAD_DELAY - -RANDOMIZE_DOWNLOAD_DELAY ------------------------- - -Default: ``True`` - -If enabled, Scrapy will wait a random amount of time (between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`) while fetching requests from the same -website. - -This randomization decreases the chance of the crawler being detected (and -subsequently blocked) by sites which analyze requests looking for statistically -significant similarities in the time between their requests. - -The randomization policy is the same used by `wget`_ ``--random-wait`` option. - -If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. - -.. _wget: https://www.gnu.org/software/wget/manual/wget.html - .. setting:: REACTOR_THREADPOOL_MAXSIZE REACTOR_THREADPOOL_MAXSIZE diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d13733623..845d7ab47 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -502,6 +502,25 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a ``robots.txt`` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed ``robots.txt`` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the ``robots.txt`` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst new file mode 100644 index 000000000..34b4f520b --- /dev/null +++ b/docs/topics/throttling.rst @@ -0,0 +1,1016 @@ +.. _throttling: + +========== +Throttling +========== + +.. versionadded:: VERSION + +Sending too many requests too quickly can `overwhelm websites`_. +:ref:`Throttling ` and :ref:`backoff ` aim to +prevent that. + +.. _overwhelm websites: https://en.wikipedia.org/wiki/Denial-of-service_attack + +.. _basic-throttling: + +Concurrency and delay +===================== + +Requests are throttled on a **per-domain basis** by default [1]_. This allows +efficient crawling of multiple sites simultaneously. + +Each domain and subdomain is treated separately: requests to +``books.toscrape.com`` and ``quotes.toscrape.com`` each have their own +throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. + +The main throttling :ref:`settings ` are: + +- .. setting:: THROTTLER_SCOPE_CONCURRENCY + + :setting:`THROTTLER_SCOPE_CONCURRENCY` (default: ``1``) + + Default maximum number of simultaneous requests per :ref:`throttler scope + `. Requests are grouped by domain by default, so this is + the maximum number of simultaneous requests per domain. + + It defines a number of “slots” per scope. Each slot can send 1 request at a + time: it sends a request, waits for the response, then sends the next + request, and so on. + +- .. setting:: DOWNLOAD_DELAY + + :setting:`DOWNLOAD_DELAY` (default: ``1`` + (:ref:`fallback `: ``0``)) + + Minimum seconds between any two requests to the same domain. + + To target a specific number of requests per minute (RPM) *per domain*, set + this to ``60 / RPM``. For example, ``DOWNLOAD_DELAY = 1.0`` for 60 RPM, or + ``DOWNLOAD_DELAY = 2.0`` for 30 RPM. + +When configuring these settings, note that: + +- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLER_SCOPE_CONCURRENCY`. + +- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, + because the next request to the domain is not sent until the delay elapses, + by which time the previous response has already arrived. + +.. [1] You can :ref:`customize ` how requests are grouped + for throttling, but domain-based throttling works well in most cases. For + more complex domain grouping strategies, see + :ref:`alternative-domain-throttling`. + +.. setting:: THROTTLER_SCOPES + +.. _per-domain-throttling: + +Per-domain throttling +===================== + +The :setting:`THROTTLER_SCOPES` setting allows you to customize throttling +behavior for specific domains [1]_. + +It is a dict that maps scope names to +:class:`~scrapy.throttler.ThrottlerScopeConfig` dicts. It is empty by default. + +:command:`startproject` scaffolds a commented-out example entry, so that you can +uncomment and edit it to crawl domains you own (or that are meant for scraping) +faster, while the :ref:`conservative defaults ` still apply to +other domains: + +.. code-block:: python + + THROTTLER_SCOPES = { + "example.com": {"concurrency": 16, "delay": 0.1}, + } + +Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are +covered later on. + +.. _backoff: + +Backoff +======= + +When servers respond with rate limiting errors (like HTTP 429) or network +timeouts occur, request rate is automatically reduced using `exponential +backoff`_. + +.. _exponential backoff: https://en.wikipedia.org/wiki/Exponential_backoff + +The key settings are: + +- .. setting:: BACKOFF_HTTP_CODES + + :setting:`BACKOFF_HTTP_CODES` (default: ``[429, 502, 503, 504, 520, 521, 522, 523, 524]``) + + HTTP response status codes that trigger backoff. Can be overridden per + scope with the ``http_codes`` key (see :ref:`per-scope-backoff`). + +- .. setting:: BACKOFF_DELAY_FACTOR + + :setting:`BACKOFF_DELAY_FACTOR` (default: ``2.0``) + + Factor by which the delay of a scope is multiplied on each backoff step + (2×, 4×, 8×, etc.). + +- .. setting:: BACKOFF_MAX_DELAY + + :setting:`BACKOFF_MAX_DELAY` (default: ``300.0``) + + Maximum delay, in seconds, that backoff can reach. Also caps + :ref:`Retry-After ` and :ref:`RateLimit-Reset + ` delays. + +See :ref:`throttling-settings` for :setting:`BACKOFF_EXCEPTIONS`, +:setting:`BACKOFF_JITTER`, :setting:`BACKOFF_MIN_DELAY` and +:setting:`BACKOFF_WINDOW`. + +.. _backoff-algorithm: + +How backoff works +----------------- + +Every :ref:`throttler scope ` keeps a current delay that +starts at its configured ``"delay"`` (:setting:`DOWNLOAD_DELAY` by default). + +A **backoff trigger** is a response whose status code is in +:setting:`BACKOFF_HTTP_CODES` or a download exception whose type is in +:setting:`BACKOFF_EXCEPTIONS`. Both can be :ref:`overridden per scope +` (through the ``http_codes`` and ``exceptions`` keys), so +the same response or exception may trigger backoff for one scope and not for +another. On each trigger: + +#. The delay grows exponentially: + + .. code-block:: text + + delay = min(BACKOFF_MAX_DELAY, max(BACKOFF_MIN_DELAY, delay * BACKOFF_DELAY_FACTOR)) + + :setting:`BACKOFF_JITTER` is then applied so that requests that backed off + together do not retry in lockstep. + +#. If the response carries a :ref:`Retry-After or RateLimit-Reset + ` value, the scope is *also* held back until that + time (capped at :setting:`BACKOFF_MAX_DELAY`) before its next request. This + is a one-time gate, on top of the exponential step above: it honors the + header for the next request without turning a short header value into a + long-standing delay for every later request. + +**Recovery** is linear: after a scope goes a full :setting:`BACKOFF_WINDOW` +without a new trigger, its delay drops by one :setting:`BACKOFF_DELAY_FACTOR` +step toward the configured value, and keeps dropping one step per quiet window +until it is back to the configured value. A new trigger resets the countdown. + +This exponential-increase / linear-decrease pattern, similar to TCP congestion +control, makes a scope back off quickly when a server is unhappy and return to +full speed gradually once it recovers. + +Backoff only ever *tightens* a scope, and recovery never goes past the +configured value: the delay can grow above the configured ``"delay"`` and then +recover back down to it, but never below it, and backoff never raises the +concurrency limit. So set the ``"delay"`` and ``"concurrency"`` you actually +want for a scope; backoff makes things gentler from there when a server pushes +back, and returns to those values once it recovers. + +Backoff triggers are detected by the +:class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, a built-in +:ref:`downloader middleware ` enabled by default. +Any component can also trigger backoff programmatically for arbitrary scopes — +e.g. based on the response body of a specific site — through +:meth:`crawler.throttler.back_off() +`. + +.. _per-scope-backoff: + +Per-scope backoff configuration +------------------------------- + +The global ``BACKOFF_*`` settings can be overridden per scope with the +``"backoff"`` key of a :setting:`THROTTLER_SCOPES` entry, an instance of +:class:`~scrapy.throttler.BackoffConfig`: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "example.com": { + "backoff": { + "http_codes": [429, 503], + "exceptions": ["builtins.IOError"], + "delay_factor": 1.2, + "max_delay": 180.0, + "min_delay": 5.0, + "jitter": [0.01, 0.33], + }, + }, + } + +Every key overrides the matching global ``BACKOFF_*`` setting for that scope +(``http_codes`` overrides :setting:`BACKOFF_HTTP_CODES`, ``exceptions`` +overrides :setting:`BACKOFF_EXCEPTIONS`, ``delay_factor`` overrides +:setting:`BACKOFF_DELAY_FACTOR`, and so on), and any key left out falls back to +it. So a scope can, for example, treat an extra status code as a backoff +trigger, or stop treating one of the defaults as a trigger, independently of +every other scope. + +.. _retry-after: +.. _rate-limiting-headers: + +Rate limiting headers +===================== + +Servers may include `Retry-After +`__ +or `RateLimit-Reset +`__ +headers to indicate when you should make your next request. These headers are +respected automatically during :ref:`backoff `: the scope's next +request is held back until the indicated time (capped at +:setting:`BACKOFF_MAX_DELAY`), on top of the usual exponential backoff step. + +.. seealso:: :setting:`REDIRECT_MAX_DELAY` + +.. _crawl-delay: + +robots.txt +========== + +`Crawl-Delay `__ +is a non-standard ``robots.txt`` directive that indicates a number of seconds +to wait between requests. + +.. setting:: THROTTLER_ROBOTSTXT_OBEY +.. setting:: THROTTLER_ROBOTSTXT_MAX_DELAY + +If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLER_ROBOTSTXT_OBEY` are +``True`` (default), valid ``Crawl-Delay`` directives override +:setting:`THROTTLER_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. +Concurrency is set to ``1`` and the delay is raised to at least the +``Crawl-Delay`` value (a larger configured delay is kept), capped at +:setting:`THROTTLER_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). + +If :setting:`THROTTLER_SCOPES` defines a different concurrency or delay, it +will be respected, but a warning will be logged about the discrepancy with +``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. + +.. _delay-scope: + +Delaying a scope programmatically +================================= + +You can delay a :ref:`throttler scope ` on demand through +:meth:`crawler.throttler.delay_scope() +`: + +.. skip: next + +.. code-block:: python + + crawler.throttler.delay_scope("example.com", 30.0) + +This holds back the scope's next request for at least the given number of +seconds and registers a :ref:`backoff ` trigger. Like a +``Retry-After`` header, it is a one-time delay rather than a permanent one (the +scope's delay also grows by one backoff step and then recovers); call it again, +e.g. on each matching response, to keep a scope slowed down for longer. + +It is useful to react to situations that :ref:`automatic backoff ` +cannot detect on its own, such as a soft block that comes back as a ``200`` +response. For example, a spider callback can slow down the whole domain when it +detects a maintenance page, and reschedule the current request: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.utils.httpobj import urlparse_cached + + + class MySpider(Spider): + name = "myspider" + start_urls = ["https://example.com/"] + + def parse(self, response): + if "under maintenance" in response.text: + scope = urlparse_cached(response).netloc + self.crawler.throttler.delay_scope(scope, 600.0) + yield response.request.replace(dont_filter=True) + return + # Normal parsing follows. + +Unlike :ref:`untrusted delays `, this delay is **not** +capped at :setting:`BACKOFF_MAX_DELAY`. + +.. _per-request-throttling: + +Per-request throttling +====================== + +Sometimes you need different throttling behavior for individual requests or for +request groups that are not tied to a specific domain. + +For example, you might want to throttle API endpoints differently than web +pages on the same domain, group requests by content type (images vs HTML), or +apply different throttling based on request priority. + +.. reqmeta:: throttler_scopes + +Use the ``throttler_scopes`` request metadata to assign requests to custom +throttling groups: + +.. invisible-code-block: python + + from scrapy.http import Request + +.. code-block:: python + + Request("https://api.example/", meta={"throttler_scopes": "api"}) + +You can also assign multiple throttling groups to a single request: + +.. code-block:: python + + Request("https://api.example/users", meta={"throttler_scopes": {"api", "users"}}) + +You can then use the :setting:`THROTTLER_SCOPES` setting to customize +throttling for such requests: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "api": {"concurrency": 2}, + "users": {"delay": 5.0}, + } + +.. note:: These custom throttling groups persist through redirects. For + redirect-aware throttling assignment, see :ref:`custom-throttler-scopes`. + +.. reqmeta:: delay + +Delaying a single request +------------------------- + +To hold a single request for a fixed number of seconds before it is sent, +regardless of its scopes, set the ``delay`` request metadata key: + +.. code-block:: python + + Request("https://example.com/slow", meta={"delay": 5.0}) + +The delay is applied once, the first time the request reaches the throttling +gate. + +``delay`` defines only the *earliest* time the request may be sent, +not the exact time: once the delay elapses, the request still competes with +every other pending request for its scopes. If you want it sent **as soon as** +its delay elapses, give it a higher :attr:`~scrapy.Request.priority` too: + +.. code-block:: python + + Request("https://example.com/slow", meta={"delay": 5.0}, priority=1) + +Without a higher priority, a backlog of requests ahead of it in a FIFO queue +could keep it waiting well past the configured delay; a higher priority puts it +at the front of the queue, so it goes out right after its delay. + +.. reqmeta:: dont_throttle + +Excluding a request from throttling state +----------------------------------------- + +Some requests (authentication flows, one-off API calls, file downloads) should +not influence throttling state even if they get a :setting:`BACKOFF_HTTP_CODES` +response or raise a :setting:`BACKOFF_EXCEPTIONS` exception. Set the +:reqmeta:`dont_throttle` request metadata key to ``True`` to process such a +request normally without letting its outcome trigger :ref:`backoff `: + +.. code-block:: python + + Request("https://example.com/login", meta={"dont_throttle": True}) + +.. _throttler-scopes: + +Throttler scopes +================ + +Throttler scopes represent aspects of requests that can be throttled +independently. + +.. + For future reference, the “throttler scope” name was taken from + https://www.ietf.org/archive/id/draft-polli-ratelimit-headers-02.html#section-1.4-4.4 + +.. _custom-throttler-scopes: + +Customizing throttler scopes +------------------------------ + +There are 2 ways to customize throttler scopes. + +To **configure existing scopes**, use the :setting:`THROTTLER_SCOPES` setting. +Its keys are scope names and its values are +:class:`~scrapy.throttler.ThrottlerScopeConfig` dicts, which accept the +following keys: + +``concurrency`` (:class:`int`) + Maximum number of concurrent requests for the scope. Defaults to + :setting:`THROTTLER_SCOPE_CONCURRENCY`. + +``delay`` (:class:`float`) + Minimum seconds between requests for the scope. Defaults to + :setting:`DOWNLOAD_DELAY`. + +``jitter`` (:class:`float` or 2-:class:`list`) + Magnitude of the random variation applied to ``delay``; the per-scope + override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. ``0`` disables it, ``0.5`` + means ±50% (the default when :setting:`RANDOMIZE_DOWNLOAD_DELAY` is on), and + a ``[low, high]`` pair multiplies the delay by ``1 + uniform(low, high)``. + +``quota`` (:class:`float`) + Maximum :ref:`quota ` consumed per ``window``. + +``window`` (:class:`float`) + Quota window in seconds. Defaults to :setting:`THROTTLER_WINDOW`. + +``backoff`` (:class:`~scrapy.throttler.BackoffConfig`) + Per-scope :ref:`backoff overrides `. + +``manager`` (:class:`str` or :class:`type`) + Import path or class of a :ref:`custom scope manager + ` for the scope. + +``ignore_robots_txt`` (:class:`bool`) + Silences the warning logged when this configuration is more aggressive than + a :ref:`robots.txt Crawl-delay `. + +.. setting:: THROTTLER + +To **change how scopes are assigned** (or anything beyond per-scope settings), +set :setting:`THROTTLER` (default: +:class:`~scrapy.throttler.Throttler`) to a :ref:`component +` that implements the +:class:`~scrapy.throttler.ThrottlerProtocol` protocol (or its import +path as a string): + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLER = "myproject.throttling.MyThrottler" + +.. _multiple-throttler-scopes: + +Handling of multiple throttler scopes +-------------------------------------- + +When a request has multiple throttler scopes, it is not sent until all of its +throttler scopes allow it. + +.. _throttler-quotas: + +Throttler quotas +---------------- + +When different requests can consume different amounts of a throttler scope, +you can express this using **throttler quotas**. + +.. setting:: THROTTLER_WINDOW + +Use the :setting:`THROTTLER_WINDOW` setting (default: ``60.0``) or the ``"window"`` +key in the :setting:`THROTTLER_SCOPES` setting to define the time window after +which throttler quotas are reset. + +Then use the :setting:`THROTTLER_SCOPES` setting to define the throttling +quotas for each throttler scope: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "api.toscrape.com": { + "quota": 500.0, + }, + } + +Then, in the :reqmeta:`throttler_scopes` request metadata key or in the return +value of the :meth:`~scrapy.throttler.ThrottlerProtocol.get_scopes` +method, define a :class:`dict` where keys are throttler scopes and values are +:class:`float` values that indicate the expected quota consumption (it does not +need to be exact). + +.. _custom-throttler-scope-managers: + +Customizing throttler scope managers +------------------------------------- + +.. setting:: THROTTLER_SCOPE_MANAGER + +The :setting:`THROTTLER_SCOPE_MANAGER` setting (default: +:class:`~scrapy.throttler.ThrottlerScopeManager`) is a :ref:`component +` that implements the +:class:`~scrapy.throttler.ThrottlerScopeManagerProtocol` (or its import path +as a string): + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPE_MANAGER = "myproject.throttling.MyThrottlerScopeManager" + +For each throttler scope, an instance of this class is created to manage any +gradual :ref:`backoff ` required at run time. + +You can implement your own throttler scope manager if you wish to change the +backoff behavior beyond what settings allow. + +You can also define a custom throttler scope manager for a specific throttling +scope by setting the ``"manager"`` key in the :setting:`THROTTLER_SCOPES` +setting: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "api.toscrape.com": { + "manager": "myproject.throttling.MyThrottlerScopeManager", + }, + } + +Most custom scope managers subclass the default +:class:`~scrapy.throttler.ThrottlerScopeManager` and override only the methods +whose behavior they want to change; implementing the +:class:`~scrapy.throttler.ThrottlerScopeManagerProtocol` from scratch is also +supported. For example, this manager disables exponential :ref:`backoff +`, so a scope relies solely on its configured delay and quota: + +.. code-block:: python + :caption: ``myproject/throttling.py`` + + from scrapy.throttler import ThrottlerScopeManager + + + class FixedWindowScopeManager(ThrottlerScopeManager): + def record_backoff(self, *args, **kwargs): + pass # never back off + +.. _throttler-aware-scheduler: + +Throttling-aware scheduling +=========================== + +By default, throttling is enforced at the engine, where a request waiting on +its :ref:`throttler scopes ` holds a concurrency slot. In a +crawl that mixes heavily-throttled scopes with unthrottled ones, this can let +throttled requests starve unthrottled ones that could be sent right away +(**head-of-line blocking**; Scrapy logs a warning the first time throttled +requests start consuming the global concurrency budget while they wait). + +:class:`~scrapy.core.scheduler.ThrottlerAwareScheduler` avoids this. To enable +it: + +.. code-block:: python + :caption: ``settings.py`` + + SCHEDULER = "scrapy.core.scheduler.ThrottlerAwareScheduler" + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ThrottlerAwarePriorityQueue" + +.. autoclass:: scrapy.core.scheduler.ThrottlerAwareScheduler + +.. _throttling-examples: + +Examples +======== + +.. _alternative-domain-throttling: + +Alternative domain throttling +----------------------------- + +If you are not happy with the :ref:`default throttler scope behavior +` with regards to domains and subdomains, you can change it. + +Alternative approaches include: + +- Using the **highest-level registrable domain** as the throttler scope, + e.g. https://books.toscrape.com and https://toscrape.com both get a + ``toscrape.com`` throttler scope. + + This allows to apply the same throttling settings to all subdomains of a + registrable domain. + + For example: + + .. code-block:: python + :caption: ``settings.py`` + + import tldextract + from scrapy.throttler import Throttler, scope_cache + from scrapy.utils.httpobj import urlparse_cached + + + class MyThrottler(Throttler): + @scope_cache + async def get_scopes(self, request): + extracted = tldextract.extract(request.url) + return extracted.registered_domain or urlparse_cached(request).netloc + + + THROTTLER = MyThrottler + +- Using **multiple throttler scopes per request**, one per registrable + domain and for every higher-level subdomain, e.g. + https://books.toscrape.com and https://toscrape.com both get a + ``toscrape.com`` throttler scope, but https://books.toscrape.com also + gets a ``books.toscrape.com`` throttler scope. + + This allows to apply the same throttling settings to all subdomains of a + registrable domain, but also allows applying further restrictions on each + or on some subdomains. + + For example: + + .. code-block:: python + :caption: ``settings.py`` + + import tldextract + from scrapy.throttler import Throttler, scope_cache + from scrapy.utils.httpobj import urlparse_cached + + + class MyThrottler(Throttler): + @scope_cache + async def get_scopes(self, request): + extracted = tldextract.extract(request.url) + if not extracted.registered_domain: + return urlparse_cached(request).netloc + # The registrable domain, plus the full host for a subdomain. + return {extracted.registered_domain, extracted.fqdn} + + + THROTTLER = MyThrottler + THROTTLER_SCOPES = { + "toscrape.com": {"concurrency": 32}, + "books.toscrape.com": {"concurrency": 24}, + "quotes.toscrape.com": {"concurrency": 16}, + } + + Here ``books.toscrape.com`` requests can reach 24 concurrency and + ``quotes.toscrape.com`` requests can reach 16 concurrency, but never both + at the same time, because that would sum 40 concurrency, and + ``toscrape.com`` requests are limited to 32. + +.. _endpoints-throttling: + +Endpoint-specific throttling +---------------------------- + +To apply different throttling settings to different endpoints of the same +domain and not enforce any common throttling, effectively treating them as +different domains: + +- Implement a :ref:`throttler ` that sets + endpoint-specific throttler scopes for that domain: + + .. code-block:: python + + from scrapy.throttler import Throttler, scope_cache + from scrapy.utils.httpobj import urlparse_cached + + + class MyThrottler(Throttler): + @scope_cache + async def get_scopes(self, request): + parsed_url = urlparse_cached(request) + if parsed_url.netloc != "api.toscrape.com": + return await super().get_scopes(request) + return f"{parsed_url.netloc}{parsed_url.path}" + +- Use the :setting:`THROTTLER_SCOPES` setting to set different throttling + settings per endpoint: + + .. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "api.toscrape.com/fast-endpoint": {"concurrency": 1000, "delay": 0.08}, + "api.toscrape.com/slow-endpoint": {"delay": 5.0}, + } + + +.. _web-scraping-api-throttling: + +Web scraping API throttling +--------------------------- + +Imagine you are sending requests to a web scraping API, e.g. to avoid bans. +Unless that API provides a Scrapy plugin to make it easier to use, you may want +to: + +- Use the :setting:`THROTTLER_SCOPES` setting to increase concurrency for + API requests. For example: + + .. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "api.toscrape.com": {"concurrency": 1000, "delay": 0.08}, + } + +- Implement a :ref:`throttler ` that: + + - Adds a throttler scope for the URL being scraped. + + For example, if you request + ``https://api.toscrape.com/?url=https://example.com``, by default it + will get a ``api.toscrape.com`` throttler scope, but it should also + get the ``example.com`` throttler scope: + + .. code-block:: python + + from urllib.parse import urlparse + + from scrapy.throttler import add_scope, Throttler, scope_cache + from scrapy.utils.httpobj import urlparse_cached + from w3lib.url import url_query_parameter + + + class MyThrottler(Throttler): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + if urlparse_cached(request).netloc != "api.toscrape.com": + return scopes + target_url = url_query_parameter(request.url, "url") + if not target_url: + return scopes + target_domain = urlparse(target_url).netloc + return add_scope(scopes, target_domain) + +- Add a :ref:`downloader middleware ` that + differentiates between exhaustion of the target website and exhaustion of + the API itself. The API returns ``200`` even when the target website + rate-limits it, reporting the upstream status in a header; the middleware + backs off the **target-website** scope (not the API scope) in that case, + checking the upstream status against :setting:`BACKOFF_HTTP_CODES`: + + .. code-block:: python + + from scrapy.throttler import iter_scopes + from scrapy.utils.httpobj import urlparse_cached + + + class UpstreamBackoffMiddleware: + def __init__(self, crawler): + self.throttler = crawler.throttler + self.backoff_codes = { + int(code) for code in crawler.settings.getlist("BACKOFF_HTTP_CODES") + } + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def process_response(self, request, response, spider): + if urlparse_cached(request).netloc == "api.toscrape.com": + upstream_status = int( + response.headers.get("X-Upstream-Status-Code", b"200") + ) + if upstream_status in self.backoff_codes: + scopes = [ + scope + for scope in iter_scopes( + self.throttler.get_resolved_scopes(request) + ) + if scope != "api.toscrape.com" + ] + self.throttler.back_off(scopes) + return response + + +.. _cost-smoothing-throttling: + +Cost-capped throttling +---------------------- + +Imagine you are using an API that charges different requests differently, e.g. +based on the features used, and you want to limit how much you spend per time +window (:setting:`THROTTLER_WINDOW`). You can use :ref:`throttler quotas +` for that: + +- Implement a :ref:`throttler ` that: + + - Sets a ``cost`` throttler scope on each request to some estimation + based e.g. on request URL parameters: + + .. code-block:: python + + from scrapy.utils.httpobj import urlparse_cached + from scrapy.throttler import Throttler, scope_cache + + + class MyThrottler(Throttler): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + parsed_url = urlparse_cached(request) + if parsed_url.netloc != "api.toscrape.com": + return scopes + return add_scope(scopes, "cost", estimate_request_cost(request)) + +- Add a :ref:`downloader middleware ` that + reconciles the estimated cost with the actual cost reported by the + response, so that the quota tracks real spending: + + .. code-block:: python + + class CostReconcileMiddleware: + def __init__(self, crawler): + self.throttler = crawler.throttler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def process_response(self, request, response, spider): + if response.headers.get("X-Actual-Cost") is not None: + estimated = estimate_request_cost(request) + actual = float(response.headers[b"X-Actual-Cost"]) + # Report the difference between actual and estimated cost. + self.throttler.reconcile_quota("cost", consumed=actual - estimated) + return response + +- Use the :setting:`THROTTLER_SCOPES` setting to set a maximum cost per time + window: + + .. code-block:: python + :caption: ``settings.py`` + + THROTTLER_SCOPES = { + "cost": {"quota": 100.0}, + } + + This will allow you to spend up to 100.0 units of cost per time window + (default: 60 seconds) before throttling kicks in. + +.. _throttling-per-ip: + +Per-IP concurrency limiting +--------------------------- + +A concurrency limit keyed by IP is just a throttler scope whose id is the +request's IP, with a ``concurrency`` limit. A request then carries two scopes, +its domain and its IP, and is only sent when **both** allow it (see +:ref:`multiple-throttler-scopes`). + +- Implement a :ref:`throttler ` that adds + the request's IP as a second scope: + + .. code-block:: python + + import socket + + from scrapy.throttler import Throttler, add_scope, scope_cache + from scrapy.utils.asyncio import run_in_thread + from scrapy.utils.httpobj import urlparse_cached + + + class IPThrottler(Throttler): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + host = urlparse_cached(request).hostname + address = await run_in_thread(socket.gethostbyname, host) + return add_scope(scopes, address) + +.. _throttling-settings: + +Additional settings +=================== + +- .. setting:: BACKOFF_ENABLED + + :setting:`BACKOFF_ENABLED` (default: ``True``) + + Whether to enable the :class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, + which drives :ref:`backoff ` from download outcomes. Set it to + ``False`` to disable backoff without having to remove the middleware from + :setting:`DOWNLOADER_MIDDLEWARES`. + +- .. setting:: BACKOFF_EXCEPTIONS + + :setting:`BACKOFF_EXCEPTIONS` + + Default: + + - :exc:`~scrapy.exceptions.DownloadFailedError` + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + - :exc:`~scrapy.exceptions.ResponseDataLossError` + + List of exception classes that trigger backoff when raised while + downloading a request. Strings are interpreted as import paths. Can be + overridden per scope with the ``exceptions`` key (see + :ref:`per-scope-backoff`). + + .. seealso:: :setting:`RETRY_EXCEPTIONS` + +- .. setting:: BACKOFF_JITTER + + :setting:`BACKOFF_JITTER` (default: ``0.1``) + + Random jitter applied to each backoff delay, as a fraction of the delay. + With the default value of ``0.1`` the delay is randomized by ±10%. + Overrides :setting:`RANDOMIZE_DOWNLOAD_DELAY` during backoff. + +- .. setting:: BACKOFF_MIN_DELAY + + :setting:`BACKOFF_MIN_DELAY` (default: ``1.0``) + + Delay, in seconds, applied on the first backoff step (and the minimum + delay during backoff). + +- .. setting:: BACKOFF_WINDOW + + :setting:`BACKOFF_WINDOW` (default: ``60.0``) + + Time window, in seconds, used by :ref:`backoff `. A + :ref:`throttler scope ` must go this many seconds + without a new backoff + trigger (an HTTP error code from :setting:`BACKOFF_HTTP_CODES` or an + exception from :setting:`BACKOFF_EXCEPTIONS`) before its delay decreases + by one :setting:`BACKOFF_DELAY_FACTOR` step toward the configured value. + A new trigger resets the countdown. + +- .. setting:: RANDOMIZE_DOWNLOAD_DELAY + + :setting:`RANDOMIZE_DOWNLOAD_DELAY` (default: ``True``) + + Randomize delays by this factor, e.g. if ``0.2`` randomize delays between + ``delay*0.8`` and ``delay*1.2``. + + It can be set to a 2-item list with low and high factors, e.g. + ``[-0.1, 0.3]`` to randomize delays between ``delay*0.9`` and + ``delay*1.3``. + + If ``True``, ``0.5`` (i.e. ±50%) is used as the randomization factor. If + ``False``, no randomization is applied. + +- .. setting:: THROTTLER_DEBUG + + :setting:`THROTTLER_DEBUG` (default: ``False``) + + Whether to log :ref:`throttling ` decisions (per-scope delays, + backoff steps and recoveries) for debugging. + +- .. setting:: THROTTLER_SCOPE_LIMIT + + :setting:`THROTTLER_SCOPE_LIMIT` (default: ``100000``) + + Maximum number of :ref:`throttler scope ` states kept + in memory at once, to bound memory usage on broad crawls that touch a large + number of scopes (e.g. domains). + + When the limit is exceeded, the least-recently-used idle scopes are evicted + (an evicted scope is recreated from its configuration the next time it is + needed). Scopes with in-flight requests or in active backoff are never + evicted, so the limit may be temporarily exceeded if that many scopes are + busy at once. Set to ``0`` to disable the limit. + + This complements :setting:`THROTTLER_SCOPE_MAX_IDLE`, which evicts scopes + by inactivity time rather than by count. + +- .. setting:: THROTTLER_SCOPE_MAX_IDLE + + :setting:`THROTTLER_SCOPE_MAX_IDLE` (default: ``3600.0``) + + Seconds of inactivity after which the state of a :ref:`throttler scope + ` is evicted from memory to bound memory usage on + long-running crawls. Set to ``0`` to never evict. Scopes in active backoff + are never evicted. + +.. _throttling-api: + +API +=== + +.. autoclass:: scrapy.throttler.ThrottlerProtocol + :members: + :member-order: bysource + +.. autoclass:: scrapy.throttler.Throttler + +.. autoclass:: scrapy.throttler.ThrottlerScopeManagerProtocol + :members: + :member-order: bysource + +.. autoclass:: scrapy.throttler.ThrottlerScopeManager + +.. autoclass:: scrapy.pqueues.ThrottlerAwarePriorityQueue + +.. autoclass:: scrapy.throttler.ThrottlerScopeConfig + +.. autoclass:: scrapy.throttler.BackoffConfig + +.. autofunction:: scrapy.throttler.scope_cache +.. autofunction:: scrapy.throttler.add_scope +.. autofunction:: scrapy.throttler.iter_scopes diff --git a/pyproject.toml b/pyproject.toml index b8d9067f9..560aff12e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "parsel>=1.5.0", "protego>=0.1.15", "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", + "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", "w3lib>=1.17.0", diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..ca3162bc1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,48 +1,39 @@ from __future__ import annotations import random +import warnings from collections import deque +from collections.abc import Iterator, Mapping from dataclasses import dataclass, field from datetime import datetime -from time import monotonic from typing import TYPE_CHECKING, Any -from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure +from twisted.internet.defer import inlineCallbacks from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.resolver import dnscache -from scrapy.utils.asyncio import ( - AsyncioLoopingCall, - CallLaterResult, - call_later, - create_looping_call, -) +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.defer import ( - _defer_sleep_async, - _schedule_coro, - deferred_from_coro, - maybe_deferred_to_future, -) -from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute +from scrapy.utils.defer import _defer_sleep_async, deferred_from_coro +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from collections.abc import Generator - from twisted.internet.task import LoopingCall + from twisted.internet.defer import Deferred from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager + from scrapy.throttler import ThrottlerProtocol, ThrottlerScopeManagerProtocol + from scrapy.utils.asyncio import CallLaterResult @dataclass(slots=True, eq=False) -class Slot: +class _Slot: """Downloader slot""" concurrency: int @@ -80,46 +71,151 @@ class Slot: ) -def _get_concurrency_delay( - concurrency: int, spider: Spider, settings: BaseSettings -) -> tuple[int, float]: - delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): - delay = spider.download_delay +Slot = create_deprecated_class( + "Slot", + _Slot, + old_class_path="scrapy.core.downloader.Slot", + subclass_warn_message=("{cls} inherits from the deprecated Slot class."), + instance_warn_message=("The Slot class is deprecated."), +) - if hasattr(spider, "max_concurrent_requests"): # pragma: no cover - warn_on_deprecated_spider_attribute( - "max_concurrent_requests", "CONCURRENT_REQUESTS" + +class _DeprecatedSlotView: + """Deprecated per-domain slot view backed by the downloader and throttler.""" + + __slots__ = ("_downloader", "_key", "_scope") + + def __init__( + self, + downloader: Downloader, + key: str, + scope: ThrottlerScopeManagerProtocol, + ) -> None: + self._downloader = downloader + self._key = key + self._scope = scope + + @property + def active(self) -> set[Request]: + return { + r + for r in self._downloader.active + if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key + } + + @property + def transferring(self) -> set[Request]: + return { + r + for r in self._downloader._transferring + if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key + } + + # This deprecated view reads throttler scope state from private attributes + # of the default scope manager rather than through the scope manager + # protocol: these are read-only compatibility accessors, so keeping them off + # the protocol avoids forcing custom THROTTLER_SCOPE_MANAGER implementations + # to provide members that only exist to feed this shim. A custom manager that + # lacks the attribute simply falls back to the historical default. + @property + def lastseen(self) -> float: + return getattr(self._scope, "_last_seen", None) or 0.0 + + @property + def delay(self) -> float: + return getattr(self._scope, "_delay", 0.0) + + @delay.setter + def delay(self, value: float) -> None: + self._scope.set_base_delay(value, only_increase=False) + + @property + def randomize_delay(self) -> bool: + return bool(getattr(self._scope, "_jitter", None)) + + @property + def concurrency(self) -> int: + warnings.warn( + "Slot.concurrency is deprecated. Per-slot concurrency limits are " + "now managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, ) - concurrency = spider.max_concurrent_requests + return getattr(self._scope, "_concurrency", None) or 0 - return concurrency, delay + def free_transfer_slots(self) -> int: + concurrency = getattr(self._scope, "_concurrency", None) or 0 + return concurrency - len(self.transferring) + + def download_delay(self) -> float: + delay = self.delay + if self.randomize_delay: + return random.uniform(0.5 * delay, 1.5 * delay) # noqa: S311 + return delay + + def close(self) -> None: + pass + + def __repr__(self) -> str: + return f"_DeprecatedSlotView({self._key!r})" + + +class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): + """Deprecated mapping view of active downloads, keyed by slot name.""" + + __slots__ = ("_downloader", "_throttler") + + def __init__(self, downloader: Downloader, throttler: ThrottlerProtocol) -> None: + self._downloader = downloader + self._throttler = throttler + + def _active_keys(self) -> set[str]: + return { + r.meta[Downloader.DOWNLOAD_SLOT] + for r in self._downloader.active + if Downloader.DOWNLOAD_SLOT in r.meta + } + + def __getitem__(self, key: str) -> _DeprecatedSlotView: + if key not in self._active_keys(): + raise KeyError(key) + scope = self._throttler.get_scope_manager(key) + return _DeprecatedSlotView(self._downloader, key, scope) + + def __iter__(self) -> Iterator[str]: + return iter(self._active_keys()) + + def __len__(self) -> int: + return len(self._active_keys()) + + def __contains__(self, key: object) -> bool: + return key in self._active_keys() class Downloader: DOWNLOAD_SLOT = "download_slot" - _SLOT_GC_INTERVAL: float = 60.0 # seconds def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals - self.slots: dict[str, Slot] = {} self.active: set[Request] = set() + self._transferring: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") - self.domain_concurrency: int = self.settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" - ) - self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") - self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) - self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS" ) + if self.per_slot_settings: + warnings.warn( + "The DOWNLOAD_SLOTS setting is deprecated. Use THROTTLER_SCOPES for " + "per-domain configuration instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) @inlineCallbacks @_warn_spider_arg @@ -140,94 +236,63 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency - @_warn_spider_arg - def _get_slot( - self, request: Request, spider: Spider | None = None - ) -> tuple[str, Slot]: - key = self.get_slot_key(request) - if key not in self.slots: - assert self.crawler.spider - slot_settings = self.per_slot_settings.get(key, {}) - conc = self.ip_concurrency or self.domain_concurrency - conc, delay = _get_concurrency_delay( - conc, self.crawler.spider, self.settings - ) - conc, delay = ( - slot_settings.get("concurrency", conc), - slot_settings.get("delay", delay), - ) - randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) - self.slots[key] = new_slot - self._start_slot_gc() + @property + def domain_concurrency(self) -> int: + warnings.warn( + "Downloader.domain_concurrency is deprecated. Per-domain concurrency " + "limits are now managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") - return key, self.slots[key] + @property + def randomize_delay(self) -> bool: + warnings.warn( + "Downloader.randomize_delay is deprecated. Delay randomization is now " + "managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") + + @property + def slots(self) -> _DeprecatedSlotsView: + warnings.warn( + "Downloader.slots is deprecated. Use the throttler API instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + assert self.crawler.throttler is not None + return _DeprecatedSlotsView(self, self.crawler.throttler) + + def _get_slot_key(self, request: Request) -> str: + assert self.crawler.throttler is not None + return self.crawler.throttler.get_slot_key(request) def get_slot_key(self, request: Request) -> str: + # Retained as public backward-compatible API. It mirrors the historical + # keying (an explicit download_slot wins, else the domain); the slot key + # used at run time comes from the throttler (see _get_slot_key()). meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) if meta_slot is not None: return meta_slot + return urlparse_cached(request).hostname or "" - key = urlparse_cached(request).hostname or "" - if self.ip_concurrency: - key = dnscache.get(key, key) - - return key - - # passed as download_func into self.middleware.download() in self.fetch() async def _enqueue_request(self, request: Request) -> Response: - key, slot = self._get_slot(request) + key = self._get_slot_key(request) request.meta[self.DOWNLOAD_SLOT] = key - slot.active.add(request) self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=self.crawler.spider, ) - d: Deferred[Response] = Deferred() - slot.queue.append((request, d)) - self._process_queue(slot) + return await self._download(request) + + async def _download(self, request: Request) -> Response: + self._transferring.add(request) try: - return await maybe_deferred_to_future(d) # fired in _wait_for_download() - finally: - slot.active.remove(request) - - def _process_queue(self, slot: Slot) -> None: - if slot.latercall: - # block processing until slot.latercall is called - return - - # Delay queue processing if a download_delay is configured - now = monotonic() - delay = slot.download_delay() - if delay: - penalty = delay - now + slot.lastseen - if penalty > 0: - slot.latercall = call_later(penalty, self._latercall, slot) - return - - # Process enqueued requests if there are free slots to transfer for this slot - while slot.queue and slot.free_transfer_slots() > 0: - slot.lastseen = now - request, queue_dfd = slot.queue.popleft() - _schedule_coro(self._wait_for_download(slot, request, queue_dfd)) - # prevent burst if inter-request delays were configured - if delay: - self._process_queue(slot) - break - - def _latercall(self, slot: Slot) -> None: - slot.latercall = None - self._process_queue(slot) - - async def _download(self, slot: Slot, request: Request) -> Response: - # The order is very important for the following logic. Do not change! - slot.transferring.add(request) - try: - # 1. Download the response response: Response = await self.handlers.download_request_async(request) - # 2. Notify response_downloaded listeners about the recent download - # before querying queue for next request self.signals.send_catch_log( signal=signals.response_downloaded, response=response, @@ -239,46 +304,12 @@ class Downloader: await _defer_sleep_async() raise finally: - # 3. After response arrives, remove the request from transferring - # state to free up the transferring slot so it can be used by the - # following requests (perhaps those which came from the downloader - # middleware itself) - slot.transferring.remove(request) - self._process_queue(slot) + self._transferring.discard(request) self.signals.send_catch_log( signal=signals.request_left_downloader, request=request, spider=self.crawler.spider, ) - async def _wait_for_download( - self, slot: Slot, request: Request, queue_dfd: Deferred[Response] - ) -> None: - try: - response = await self._download(slot, request) - except Exception: - queue_dfd.errback(Failure()) - else: - queue_dfd.callback(response) # awaited in _enqueue_request() - def close(self) -> None: - self._stop_slot_gc() - for slot in self.slots.values(): - slot.close() - - def _slot_gc(self, age: float = 60) -> None: - mintime = monotonic() - age - for key, slot in list(self.slots.items()): - if not slot.active and slot.lastseen + slot.delay < mintime: - self.slots.pop(key).close() - - def _start_slot_gc(self) -> None: - if self._slot_gc_loop: - return - self._slot_gc_loop = create_looping_call(self._slot_gc) - self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False) - - def _stop_slot_gc(self) -> None: - if self._slot_gc_loop: - self._slot_gc_loop.stop() - self._slot_gc_loop = None + pass diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py index 83d130462..b0ddca604 100644 --- a/scrapy/core/downloader/handlers/_base_http.py +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -7,11 +7,34 @@ from .base import BaseDownloadHandler if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): """Base class for built-in HTTP download handlers.""" + @staticmethod + def _max_per_host_concurrency(settings: BaseSettings) -> int: + """Highest per-host concurrency the throttler may admit: the per-domain + limit, the default ``other``-scope limit, and any explicit + :setting:`THROTTLER_SCOPES` concurrency. + + Since :setting:`CONCURRENT_REQUESTS` caps the total number of requests in + flight, no host can ever exceed it, so it is also the upper bound of the + result. + """ + global_concurrency = settings.getint("CONCURRENT_REQUESTS") + candidates = [ + settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), + settings.getint("THROTTLER_SCOPE_CONCURRENCY"), + ] + candidates += [ + int(scope["concurrency"]) + for scope in settings.getdict("THROTTLER_SCOPES").values() + if "concurrency" in scope + ] + return min(max(candidates), global_concurrency) + def __init__(self, crawler: Crawler): super().__init__(crawler) self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py index 5a669c732..d74a7d120 100644 --- a/scrapy/core/downloader/handlers/_base_streaming.py +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -84,11 +84,8 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon crawler.settings.get("DOWNLOAD_BIND_ADDRESS") ) self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") - # these are useful for many handlers but used in different ways by them self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") - self._pool_size_per_host: int = crawler.settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" - ) + self._pool_size_per_host: int = self._max_per_host_concurrency(crawler.settings) @staticmethod @abstractmethod diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 17dca5acb..cb46f2103 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,8 +92,8 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): from twisted.internet import reactor self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) - self._pool.maxPersistentPerHost = crawler.settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" + self._pool.maxPersistentPerHost = self._max_per_host_concurrency( + crawler.settings ) self._pool._factory.noisy = False diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1033e874f..34acd5051 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -31,6 +31,7 @@ from scrapy.exceptions import ( from scrapy.http import Request, Response from scrapy.utils.asyncio import ( AsyncioLoopingCall, + call_later, create_looping_call, is_asyncio_available, ) @@ -57,6 +58,7 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings, Settings from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider + from scrapy.utils.asyncio import CallLaterResult logger = logging.getLogger(__name__) @@ -129,6 +131,25 @@ class ExecutionEngine: self._start_request_processing_awaitable: ( asyncio.Future[None] | Deferred[None] | None ) = None + # Requests currently held by the throttler, waiting for their + # scopes to allow them through to the downloader. + self._throttler_waiting: set[Request] = set() + # Number of in-flight asynchronous enqueue operations (see + # ``_enqueue_request_async``), so the spider is not considered idle + # while a request is still on its way into the scheduler. + self._scheduling: int = 0 + # A coalesced wakeup timer, armed when the scheduler reports (through + # ``get_next_request_delay``) that every pending request is time-blocked. + self._delay_wakeup: CallLaterResult | None = None + # The scheduler's ``get_next_request_delay`` bound method, cached once + # per crawl in ``open_spider_async`` (``None`` if the scheduler does not + # expose one). Used both to arm the wakeup timer and to tell whether the + # scheduler holds time-blocked requests itself. + self._get_next_request_delay: Callable[[], float | None] | None = None + # Whether the scheduler exposes enqueue_request_async, cached once per + # crawl in ``open_spider_async``; checked for every scheduled request. + self._scheduler_enqueues_async: bool = False + self._throttler_backout_warned: bool = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -262,9 +283,16 @@ class ExecutionEngine: def pause(self) -> None: self.paused = True + self._cancel_delay_wakeup() def unpause(self) -> None: self.paused = False + # pause() cancels the coalesced delay wakeup and the loop stops + # re-running itself while paused, so re-run it here: this re-arms the + # wakeup and keeps a crawl held only by a time-based gate (nothing in + # flight to re-run the loop from _download) from stalling. + if self._slot is not None: + self._slot.nextcall.schedule() async def _process_start_next(self) -> None: """Processes the next item or request from Spider.start(). @@ -331,13 +359,46 @@ class ExecutionEngine: if self._slot is None or self._slot.closing is not None or self.paused: return + self._cancel_delay_wakeup() + while not self.needs_backout(): if not self._start_scheduled_request(): break + self._maybe_arm_delay_wakeup() + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() + def _cancel_delay_wakeup(self) -> None: + if self._delay_wakeup is not None: + self._delay_wakeup.cancel() + self._delay_wakeup = None + + def _maybe_arm_delay_wakeup(self) -> None: + """Arm a single coalesced wakeup when the scheduler reports that every + pending request is time-blocked. + + Concurrency-blocked states do not need this: a freed slot already + re-runs the loop from :meth:`_download`'s ``finally``. Only time-based + gates (delay, backoff, quota windows) need a timer, since nothing else + would re-run the loop while they are closed. + """ + assert self._slot is not None + if ( + self._get_next_request_delay is None + or not self._slot.scheduler.has_pending_requests() + ): + return + # A positive delay means a time-based gate nothing else would wake us + # for. ``None`` (nothing time-blocked) and ``0`` (something is ready but + # could not be sent, e.g. the downloader is full) are handled elsewhere: + # a freed slot re-runs the loop, and a ``0``-second timer would busy-loop. + delay = self._get_next_request_delay() + if delay is None or delay <= 0: + return + self._delay_wakeup = call_later(delay, self._slot.nextcall.schedule) + def needs_backout(self) -> bool: """Returns ``True`` if no more requests can be sent at the moment, or ``False`` otherwise. @@ -351,6 +412,42 @@ class ExecutionEngine: or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() + or self._throttler_needs_backout() + ) + + def _throttler_needs_backout(self) -> bool: + """Apply backpressure while requests are held by the throttler. + + Requests waiting on throttling do not occupy a downloader concurrency + slot, so without this check the engine would keep draining the + scheduler into pending throttling waits. Count them together with the + in-flight requests against the global concurrency limit. + """ + if not self._throttler_waiting: + return False + if ( + len(self._throttler_waiting) + len(self.downloader.active) + < self.downloader.total_concurrency + ): + return False + self._maybe_warn_throttler_backout() + return True + + def _maybe_warn_throttler_backout(self) -> None: + if self._throttler_backout_warned: + return + # A throttler-aware scheduler holds time-blocked requests itself instead + # of letting them pile up in _throttler_waiting and back-pressure the + # engine here, so the warning does not apply when one is in use. + if self._get_next_request_delay is not None: + return + self._throttler_backout_warned = True + logger.warning( + "Throttling is holding requests back and they are now consuming the " + "global concurrency budget while they wait for a free slot. Consider " + "switching to scrapy.core.scheduler.ThrottlerAwareScheduler, which " + "holds throttled requests without consuming concurrency slots.", + extra={"spider": self.spider}, ) def _remove_request(self, _: Any, request: Request) -> None: @@ -426,6 +523,10 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False + if self._throttler_waiting: # requests held by the throttler + return False + if self._scheduling: # requests still on their way into the scheduler + return False if self._start is not None: # not all start requests are handled return False return not self._slot.scheduler.has_pending_requests() @@ -438,6 +539,7 @@ class ExecutionEngine: self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request) -> None: + assert self._slot is not None request_scheduled_result = self.signals.send_catch_log( signals.request_scheduled, request=request, @@ -447,11 +549,39 @@ class ExecutionEngine: for _, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return - if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + scheduler = self._slot.scheduler + if self._scheduler_enqueues_async: + self._scheduling += 1 + _schedule_coro(self._enqueue_request_async(request)) + return + if not scheduler.enqueue_request(request): self.signals.send_catch_log( signals.request_dropped, request=request, spider=self.spider ) + async def _enqueue_request_async(self, request: Request) -> None: + # _scheduling is incremented in _schedule_request before this coroutine + # is scheduled, so it must be decremented on every path (hence finally), + # otherwise spider_is_idle() never reports idle. + try: + if self._slot is None: + return + stored = await self._slot.scheduler.enqueue_request_async(request) # type: ignore[attr-defined] + if not stored: + self.signals.send_catch_log( + signals.request_dropped, request=request, spider=self.spider + ) + except Exception: + logger.error( + "Error while enqueuing request", + exc_info=True, + extra={"spider": self.spider}, + ) + finally: + self._scheduling -= 1 + if self._slot is not None: + self._slot.nextcall.schedule() + def download(self, request: Request) -> Deferred[Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" warnings.warn( @@ -481,6 +611,20 @@ class ExecutionEngine: return await self.download_async(response_or_request) return response_or_request + @inlineCallbacks + def _acquire_throttler( + self, request: Request + ) -> Generator[Deferred[Any], Any, None]: + """Wait at the throttling gate before *request* is sent, tracking it as + held meanwhile.""" + self._throttler_waiting.add(request) + throttler = self.crawler.throttler + assert throttler is not None + try: + yield deferred_from_coro(throttler.acquire(request)) + finally: + self._throttler_waiting.discard(request) + @inlineCallbacks def _download( self, request: Request @@ -488,8 +632,14 @@ class ExecutionEngine: assert self._slot is not None # typing assert self.spider is not None - self._slot.add_request(request) + throttler = self.crawler.throttler + assert throttler is not None + # A throttler-aware scheduler reserves the request (a concurrency slot) + # before handing it here, so everything past this point runs inside the + # try/finally that releases it, even if add_request() were to raise. try: + self._slot.add_request(request) + yield self._acquire_throttler(request) result: Response | Request if self._downloader_fetch_needs_spider: result = yield self.downloader.fetch(request, self.spider) @@ -515,6 +665,7 @@ class ExecutionEngine: ) return result finally: + throttler.release(request) self._slot.nextcall.schedule() def open_spider( @@ -537,6 +688,10 @@ class ExecutionEngine: self.spider = self.crawler.spider nextcall = CallLaterOnce(self._start_scheduled_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) + self._get_next_request_delay = getattr( + scheduler, "get_next_request_delay", None + ) + self._scheduler_enqueues_async = hasattr(scheduler, "enqueue_request_async") self._slot = _Slot(close_if_idle, nextcall, scheduler) self._start = await self.scraper.spidermw.process_start() if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): @@ -613,6 +768,8 @@ class ExecutionEngine: "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) + self._cancel_delay_wakeup() + try: await self._slot.close() except Exception: diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 82e31b90b..3ffb6a78c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred # noqa: TC002 from scrapy.spiders import Spider # noqa: TC001 +from scrapy.throttler import iter_scopes from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -25,6 +26,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.pqueues import ScrapyPriorityQueue from scrapy.statscollectors import StatsCollector + from scrapy.throttler import ThrottlerProtocol logger = logging.getLogger(__name__) @@ -63,6 +65,33 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`. The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. + + Asynchronous API + ================ + + Beyond the minimal interface above, the engine also uses the following + optional members when a scheduler defines them, to support asynchronous + scheduling. :meth:`next_request` itself stays synchronous. + + .. method:: enqueue_request_async(request) + :async: + + Asynchronous counterpart of :meth:`enqueue_request`, following the same + return contract. When a scheduler defines it, the engine awaits it + *instead of* calling :meth:`enqueue_request`. Define it when enqueuing a + request needs to ``await`` (e.g. to resolve throttler scopes + asynchronously). + + .. method:: get_next_request_delay() + + Return the number of seconds until some pending request that + :meth:`next_request` is currently withholding for a time-based reason + becomes available, or ``None`` if no pending request is time-blocked. + When a scheduler defines it, the engine uses it to schedule a single + wakeup after :meth:`next_request` returns ``None`` while requests remain + pending, so a crawl held back only by time does not stall. + + .. versionadded:: VERSION """ @classmethod @@ -209,9 +238,8 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS` or - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent - concurrently. + :setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLER_SCOPE_CONCURRENCY`, + those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired order. Lowering those settings to ``1`` enforces the desired order except @@ -374,12 +402,21 @@ class Scheduler(BaseScheduler): if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False - dqok = self._dqpush(request) + return self._store(request) + + def _store(self, request: Request, *push_args: Any) -> bool: + """Push *request* into the disk queue, falling back to the memory queue, + and increment the relevant ``scheduler/enqueued*`` stats. + + Extra positional arguments are forwarded to the underlying queue + ``push`` calls; subclasses that store requests under additional keys + (e.g. a throttler scope set) pass them through here. + """ assert self.stats is not None - if dqok: + if self._dqpush(request, *push_args): self.stats.inc_value("scheduler/enqueued/disk") else: - self._mqpush(request) + self._mqpush(request, *push_args) self.stats.inc_value("scheduler/enqueued/memory") self.stats.inc_value("scheduler/enqueued") return True @@ -411,11 +448,11 @@ class Scheduler(BaseScheduler): """ return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs) - def _dqpush(self, request: Request) -> bool: + def _dqpush(self, request: Request, *push_args: Any) -> bool: if self.dqs is None: return False try: - self.dqs.push(request) + self.dqs.push(request, *push_args) except ValueError as e: # non serializable request if self.logunser: msg = ( @@ -435,8 +472,8 @@ class Scheduler(BaseScheduler): return False return True - def _mqpush(self, request: Request) -> None: - self.mqs.push(request) + def _mqpush(self, request: Request, *push_args: Any) -> None: + self.mqs.push(request, *push_args) def _dqpop(self) -> Request | None: if self.dqs is not None: @@ -496,3 +533,69 @@ class Scheduler(BaseScheduler): def _write_dqs_state(self, dqdir: str, state: Any) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) + + +class ThrottlerAwareScheduler(Scheduler): + """A :setting:`SCHEDULER` that only ever hands the engine requests that + their :ref:`throttler scopes ` allow to be sent **right + now**. + + The default scheduler hands requests to the engine as concurrency allows + and lets them wait at the throttling gate + (:meth:`~scrapy.throttler.ThrottlerProtocol.acquire`) while holding + a concurrency slot. When a crawl mixes heavily-throttled scopes with + unthrottled ones, enough throttled requests waiting on a clock can fill + :setting:`CONCURRENT_REQUESTS` and starve unthrottled requests that could be + sent right away. This scheduler instead withholds a request until it is + sendable, so held-back requests occupy neither concurrency slots nor memory + (beyond what is needed to track each distinct scope set) and cannot starve + other scopes. + + It also honors the per-request :reqmeta:`delay`, holding an + individual request back without blocking others that share its scopes, which + the default scheduler cannot do. + + When several requests could be sent at the same time, the one with the + highest request :attr:`~scrapy.Request.priority` is sent first; ties are + broken by preferring the least-busy scopes. + + It requires :setting:`SCHEDULER_PRIORITY_QUEUE` to be set to + :class:`~scrapy.pqueues.ThrottlerAwarePriorityQueue` (or a compatible + subclass). + """ + + def open(self, spider: Spider) -> Deferred[None] | None: + result = super().open(spider) + if not hasattr(self.mqs, "get_next_request_delay"): + raise ValueError( + f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be " + f"set to a throttler-aware priority queue such as " + f"scrapy.pqueues.ThrottlerAwarePriorityQueue, but the " + f"configured one ({type(self.mqs).__name__}) is not." + ) + assert self.crawler is not None + assert self.crawler.throttler is not None + self._throttler: ThrottlerProtocol = self.crawler.throttler + return result + + def enqueue_request(self, request: Request) -> bool: + raise RuntimeError( + "ThrottlerAwareScheduler requires the asynchronous enqueue path; " + "enqueue_request_async() is used by the engine instead of " + "enqueue_request()." + ) + + async def enqueue_request_async(self, request: Request) -> bool: + if not request.dont_filter and self.df.request_seen(request): + self.df.log(request, self.spider) + return False + scope_set = frozenset(iter_scopes(await self._throttler.get_scopes(request))) + return self._store(request, scope_set) + + def get_next_request_delay(self) -> float | None: + delays = [ + delay + for pq in (self.mqs, self.dqs) + if pq is not None and (delay := pq.get_next_request_delay()) is not None # type: ignore[attr-defined] + ] + return min(delays) if delays else None diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c72752915..508e6fabf 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: from scrapy.logformatter import LogFormatter from scrapy.statscollectors import StatsCollector + from scrapy.throttler import ThrottlerProtocol from scrapy.utils.request import RequestFingerprinterProtocol @@ -83,6 +84,13 @@ class Crawler: self.stats: StatsCollector | None = None self.logformatter: LogFormatter | None = None self.request_fingerprinter: RequestFingerprinterProtocol | None = None + self.throttler: ThrottlerProtocol | None = None + """The throttler of this crawler, an instance of + :setting:`THROTTLER`. + + It is ``None`` until the crawl starts. Components can use it to inspect + or drive :ref:`throttling ` at run time, e.g. through + :meth:`~scrapy.throttler.ThrottlerProtocol.delay_scope`.""" self.spider: Spider | None = None self.engine: ExecutionEngine | None = None @@ -96,6 +104,10 @@ class Crawler: return self.addons.load_settings(self.settings) + self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") + self._apply_deprecated_spider_attr( + "max_concurrent_requests", "THROTTLER_SCOPE_CONCURRENCY" + ) self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -105,6 +117,10 @@ class Crawler: load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), self, ) + self.throttler = build_from_crawler( + load_object(self.settings["THROTTLER"]), + self, + ) use_reactor = self.settings.getbool("TWISTED_REACTOR_ENABLED") if use_reactor: @@ -151,6 +167,30 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None: + """Bridge a deprecated spider attribute onto *setting*, warning about + the deprecation (and about being ignored when *setting* is already set + at spider or higher priority).""" + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, attr): + return + if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + f"The {attr!r} spider attribute is deprecated. It is also being " + f"ignored because {setting} is already set at spider or higher " + f"priority. Remove the {attr!r} attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + return + warnings.warn( + f"The {attr!r} spider attribute is deprecated. Use the {setting} " + f"setting or THROTTLER_SCOPES instead.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + self.settings.set(setting, getattr(spider, attr), priority="spider") + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py new file mode 100644 index 000000000..37a10ca21 --- /dev/null +++ b/scrapy/downloadermiddlewares/backoff.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from scrapy.exceptions import NotConfigured +from scrapy.throttler import iter_scopes +from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.misc import _load_objects + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + import scrapy + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.throttler import ThrottlerProtocol + + +logger = logging.getLogger(__name__) + + +class BackoffMiddleware: + """Downloader middleware that drives :ref:`backoff ` from download + outcomes. + + It observes every response and download exception and, for those matching + :setting:`BACKOFF_HTTP_CODES` or :setting:`BACKOFF_EXCEPTIONS` (globally or + per :setting:`THROTTLER_SCOPES` scope), tells the :ref:`throttler + ` to back off the request's scopes through its + :meth:`~scrapy.throttler.ThrottlerProtocol.back_off` API. + + It is enabled by default; set :setting:`BACKOFF_ENABLED` to ``False`` to + disable it without removing it from :setting:`DOWNLOADER_MIDDLEWARES`. + + See :ref:`throttling` for details. + """ + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("BACKOFF_ENABLED"): + raise NotConfigured + assert crawler.throttler is not None + self._throttler: ThrottlerProtocol = crawler.throttler + settings = crawler.settings + self._http_codes: set[int] = { + int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") + } + self._exceptions: tuple[type[BaseException], ...] = _load_objects( + settings.getlist("BACKOFF_EXCEPTIONS") + ) + # Per-scope overrides; a scope absent from these uses the globals above. + self._scope_http_codes: dict[str, set[int]] = {} + self._scope_exceptions: dict[str, tuple[type[BaseException], ...]] = {} + # Unions of the globals and every per-scope override, used as a cheap + # pre-filter to skip outcomes that no scope backs off on. + self._any_http_codes: set[int] = set(self._http_codes) + self._any_exceptions: tuple[type[BaseException], ...] = self._exceptions + for scope_id, scope_config in settings.getdict("THROTTLER_SCOPES").items(): + backoff = scope_config.get("backoff") or {} + if "http_codes" in backoff: + codes = {int(code) for code in backoff["http_codes"]} + self._scope_http_codes[scope_id] = codes + self._any_http_codes |= codes + if "exceptions" in backoff: + exceptions = _load_objects(backoff["exceptions"]) + self._scope_exceptions[scope_id] = exceptions + self._any_exceptions += exceptions + + @_warn_spider_arg + def process_response( + self, + request: Request, + response: Response, + spider: scrapy.Spider | None = None, + ) -> Response: + if ( + response.status not in self._any_http_codes + or "cached" in response.flags + or request.meta.get("dont_throttle") + ): + return response + matched = [ + scope + for scope in iter_scopes(self._throttler.get_resolved_scopes(request)) + if response.status in self._scope_http_codes.get(scope, self._http_codes) + ] + if matched: + self._throttler.back_off(matched, delay=self._response_delay(response)) + return response + + @_warn_spider_arg + def process_exception( + self, + request: Request, + exception: Exception, + spider: scrapy.Spider | None = None, + ) -> None: + if request.meta.get("dont_throttle") or not isinstance( + exception, self._any_exceptions + ): + return + matched = [ + scope + for scope in iter_scopes(self._throttler.get_resolved_scopes(request)) + if isinstance( + exception, self._scope_exceptions.get(scope, self._exceptions) + ) + ] + if matched: + self._throttler.back_off(matched) + + @staticmethod + def _response_delay(response: Response) -> float | None: + """Return the hard minimum delay requested by *response* through a + ``Retry-After`` or ``RateLimit-Reset`` header, or ``None``.""" + delays = [ + delay + for delay in ( + _parse_retry_after(response), + _parse_ratelimit_reset(response), + ) + if delay is not None + ] + return max(delays) if delays else None diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 45af5c67a..aa4f2cdab 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -10,6 +10,7 @@ from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.utils._headers import _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import global_object_name @@ -36,6 +37,7 @@ class BaseRedirectMiddleware: raise NotConfigured self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") + self.max_delay: float = settings.getfloat("REDIRECT_MAX_DELAY") self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") self._referer_spider_middleware: RefererMiddleware | None = None @@ -174,9 +176,30 @@ class BaseRedirectMiddleware: del redirect_request.headers["Authorization"] self.handle_referer(redirect_request, response) + self._apply_retry_after(redirect_request, response) return redirect_request + def _apply_retry_after(self, redirect_request: Request, response: Response) -> None: + """Delay the redirect when *response* carries a ``Retry-After`` header. + + The delay is capped at :setting:`REDIRECT_MAX_DELAY` and applied through + the :reqmeta:`delay` request metadata key, which holds back + this request only (without counting as a :ref:`backoff ` + trigger for its scopes). :setting:`REDIRECT_MAX_DELAY` set to ``0`` + disables it. + """ + if not self.max_delay: + return + retry_after = _parse_retry_after(response) + if retry_after is None: + return + redirect_request.meta["delay"] = min(retry_after, self.max_delay) + # This is a fresh request that may inherit an already-honored delay from + # its source request's meta; clear that state so the new delay applies. + redirect_request.meta.pop("_throttler_delayed", None) + redirect_request.meta.pop("_throttler_delay_deadline", None) + def _redirect_request_using_get( self, request: Request, response: Response, redirect_url: str ) -> Request: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 542ff1cdc..befdfc293 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -2,17 +2,19 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING +from warnings import warn from scrapy import Request, Spider, signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy.core.downloader import Slot from scrapy.crawler import Crawler from scrapy.http import Response + from scrapy.throttler import ThrottlerProtocol logger = logging.getLogger(__name__) @@ -24,6 +26,15 @@ class AutoThrottle: if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured + warn( + "You have set the AUTOTHROTTLE_ENABLED setting to True, however " + "the AutoThrottle extension is deprecated; use throttler and " + "backoff settings instead: " + "https://docs.scrapy.org/en/latest/topics/throttling.html", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" @@ -33,6 +44,7 @@ class AutoThrottle: f"AUTOTHROTTLE_TARGET_CONCURRENCY " f"({self.target_concurrency!r}) must be higher than 0." ) + self._started_scopes: set[str] = set() crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded @@ -45,7 +57,9 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] + self._startdelay = max( + self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") + ) def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings @@ -54,55 +68,51 @@ class AutoThrottle: def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider: Spider) -> float: - return max( - self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") - ) - def _response_downloaded( self, response: Response, request: Request, spider: Spider ) -> None: - key, slot = self._get_slot(request, spider) + throttler = self.crawler.throttler + assert throttler is not None latency = request.meta.get("download_latency") if ( latency is None - or slot is None or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return - olddelay = slot.delay - self._adjust_delay(slot, latency, response) + scope_id = urlparse_cached(request).hostname or "" + olddelay = self._scope_delay(throttler, scope_id) + newdelay = self._adjust_delay(olddelay, latency, response) + throttler.set_scope_delay(scope_id, newdelay) if self.debug: - diff = slot.delay - olddelay - size = len(response.body) - conc = len(slot.transferring) logger.info( - "slot: %(slot)s | conc:%(concurrency)2d | " + "slot: %(slot)s | " "delay:%(delay)5d ms (%(delaydiff)+d) | " "latency:%(latency)5d ms | size:%(size)6d bytes", { - "slot": key, - "concurrency": conc, - "delay": slot.delay * 1000, - "delaydiff": diff * 1000, + "slot": scope_id, + "delay": newdelay * 1000, + "delaydiff": (newdelay - olddelay) * 1000, "latency": latency * 1000, - "size": size, + "size": len(response.body), }, extra={"spider": spider}, ) - def _get_slot( - self, request: Request, spider: Spider - ) -> tuple[str | None, Slot | None]: - key: str | None = request.meta.get("download_slot") - if key is None: - return None, None - assert self.crawler.engine - return key, self.crawler.engine.downloader.slots.get(key) + def _scope_delay(self, throttler: ThrottlerProtocol, scope_id: str) -> float: + """Return the current delay of *scope_id*, applying AUTOTHROTTLE_START_DELAY + the first time the scope is seen.""" + delay = throttler.get_scope_delay(scope_id) + if scope_id not in self._started_scopes: + self._started_scopes.add(scope_id) + delay = max(delay, self._startdelay) + return delay - def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: - """Define delay adjustment policy""" + def _adjust_delay( + self, olddelay: float, latency: float, response: Response + ) -> float: + """Return the new delay given the current *olddelay* and the observed + *latency*.""" # If a server needs `latency` seconds to respond then # we should send a request each `latency/N` seconds @@ -110,7 +120,7 @@ class AutoThrottle: target_delay = latency / self.target_concurrency # Adjust the delay to make it closer to target_delay - new_delay = (slot.delay + target_delay) / 2.0 + new_delay = (olddelay + target_delay) / 2.0 # If target delay is bigger than old delay, then use it instead of mean. # It works better with problematic sites. @@ -123,7 +133,7 @@ class AutoThrottle: # than old one, as error pages (and redirections) are usually small and # so tend to reduce latency, thus provoking a positive feedback by # reducing delay instead of increase. - if response.status != 200 and new_delay <= slot.delay: - return + if response.status != 200 and new_delay <= olddelay: + return olddelay - slot.delay = new_delay + return new_delay diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 41411ceaf..520550874 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,7 +1,10 @@ from __future__ import annotations import hashlib +import heapq +import json import logging +import time from typing import TYPE_CHECKING, Protocol, cast from scrapy.utils.misc import build_from_crawler @@ -13,8 +16,8 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy import Request - from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler + from scrapy.throttler import ScopeID, ThrottlerProtocol logger = logging.getLogger(__name__) @@ -235,9 +238,9 @@ class ScrapyPriorityQueue: if self.curprio is None: return None try: - queue = self._start_queues[self.curprio] - except KeyError: queue = self.queues[self.curprio] + except KeyError: + queue = self._start_queues[self.curprio] # Protocols can't declare optional members return cast("Request", queue.peek()) # type: ignore[attr-defined] @@ -261,24 +264,6 @@ class ScrapyPriorityQueue: ) -class DownloaderInterface: - def __init__(self, crawler: Crawler): - assert crawler.engine - self.downloader: Downloader = crawler.engine.downloader - - def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: - return [(self._active_downloads(slot), slot) for slot in possible_slots] - - def get_slot_key(self, request: Request) -> str: - return self.downloader.get_slot_key(request) - - def _active_downloads(self, slot: str) -> int: - """Return a number of requests in a Downloader for a given slot""" - if slot not in self.downloader.slots: - return 0 - return len(self.downloader.slots[slot].active) - - class DownloaderAwarePriorityQueue: """PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued @@ -332,11 +317,6 @@ class DownloaderAwarePriorityQueue: *, start_queue_cls: type[QueueProtocol] | None = None, ): - if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: - raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' - ) - if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError( "DownloaderAwarePriorityQueue accepts " @@ -348,7 +328,8 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + assert crawler.throttler is not None + self._throttler: ThrottlerProtocol = crawler.throttler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key @@ -360,19 +341,19 @@ class DownloaderAwarePriorityQueue: for slot, startprios in slot_startprios.items(): self.pqueues[slot] = self.pqfactory(slot, startprios) - def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str: + def _next_slot(self, stats: list[tuple[float, str]], *, update_state: bool) -> str: last = self._last_selected_slot - min_active: int | None = None + min_load: float | None = None best_slot: str | None = None best_slot_after_last: str | None = None - for active, slot in stats: - if min_active is None or active < min_active: - min_active = active + for load, slot in stats: + if min_load is None or load < min_load: + min_load = load best_slot = slot best_slot_after_last = None if last is not None and slot > last: best_slot_after_last = slot - elif active == min_active: + elif load == min_load: if best_slot is None or slot < best_slot: best_slot = slot if ( @@ -398,8 +379,11 @@ class DownloaderAwarePriorityQueue: start_queue_cls=self._start_queue_cls, ) + def _slot_stats(self) -> list[tuple[float, str]]: + return [(self._throttler.get_scope_load(slot), slot) for slot in self.pqueues] + def pop(self) -> Request | None: - stats = self._downloader_interface.stats(self.pqueues) + stats = self._slot_stats() if not stats: return None @@ -412,7 +396,7 @@ class DownloaderAwarePriorityQueue: return request def push(self, request: Request) -> None: - slot = self._downloader_interface.get_slot_key(request) + slot = self._throttler.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] @@ -425,7 +409,7 @@ class DownloaderAwarePriorityQueue: Raises :exc:`NotImplementedError` if the underlying queue class does not implement a ``peek`` method, which is optional for queues. """ - stats = self._downloader_interface.stats(self.pqueues) + stats = self._slot_stats() if not stats: return None slot = self._next_slot(stats, update_state=False) @@ -442,3 +426,260 @@ class DownloaderAwarePriorityQueue: def __contains__(self, slot: str) -> bool: return slot in self.pqueues + + +def _scope_set_key(scope_set: frozenset[ScopeID]) -> str: + """Return a reversible, JSON-safe string key for *scope_set*. + + Used both as the in-memory dict key encoding for the on-disk state and to + derive the per-scope-set subdirectory name. The encoding is + order-independent (the scope ids are sorted).""" + return json.dumps(sorted(scope_set)) + + +def _scope_set_from_key(key: str) -> frozenset[ScopeID]: + return frozenset(json.loads(key)) + + +class ThrottlerAwarePriorityQueue: + """Priority queue that only ever pops a request that can be sent right now + based on its :ref:`throttler scope set ` and + per-request :reqmeta:`delay`. + + The downstream queue class must support ``peek``. + + Disk persistence + ================ + + .. warning:: The files that this class generates on disk are an + implementation detail, and may change without a warning in a future + version of Scrapy. Do not rely on the following information for + anything other than debugging purposes. + + When instantiated with a non-empty *key* argument, *key* is used as a + persistence directory, and inside it this class creates a subdirectory per + scope set, named from a path-safe, order-independent encoding of its scope + ids. + + For example, a request whose scope set is ``{"example.com", + "cost:group-1"}`` is stored under a subdirectory derived in two steps: + + #. The scope ids are sorted and JSON-encoded into an order-independent key + (so ``{"example.com", "cost:group-1"}`` and ``{"cost:group-1", + "example.com"}`` map to the same one):: + + ["cost:group-1", "example.com"] + + #. That key is made path-safe: every character outside ``[A-Za-z0-9-._]`` + becomes ``_`` (here the ``:`` and the JSON quotes, brackets and spaces; + the ``-`` and ``.`` are kept), and an MD5 suffix disambiguates keys that + collapse to the same path:: + + __cost_group-1____example.com__-fc6ba2aff8f421bf981b662d77739902 + """ + + @classmethod + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, + ) -> Self: + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) + + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + slot_startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, + ): + if slot_startprios and not isinstance(slot_startprios, dict): + raise ValueError( + "ThrottlerAwarePriorityQueue accepts ``slot_startprios`` as a " + f"dict; {slot_startprios.__class__!r} instance is passed. Most " + "likely, it means the state is created by an incompatible " + "priority queue. Only a crawl started with the same priority " + "queue class can be resumed." + ) + + assert crawler.throttler is not None + self._throttler: ThrottlerProtocol = crawler.throttler + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls + self.key: str = key + self.crawler: Crawler = crawler + + self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {} + if slot_startprios: + for set_key, startprios in slot_startprios.items(): + scope_set = _scope_set_from_key(set_key) + self.pqueues[scope_set] = self._pqfactory(scope_set, startprios) + + # Requests held back by their own per-request delay wait + # here instead of in their scope-set queue, so a not-yet-due request + # never sits at a queue head and blocks the other requests that share + # its scopes (the head-of-line blocking this scheduler exists to + # avoid). Once the delay elapses, _promote_ready() moves the request + # into its scope-set queue, where it competes normally. + # + # This is a min-heap of (deadline, seq, scope_set, request). heapq + # needs a total order and always compares entries to keep its invariant; + # seq (a monotonic counter) makes the (deadline, seq) prefix totally + # ordered, so a deadline tie is broken there and the scope_set (a + # frozenset, whose < is only the partial subset order) and the request + # (no ordering at all) are never compared. The order among tied + # deadlines is irrelevant beyond being deterministic. + self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] + self._delayed_seq: int = 0 + + def _pqfactory( + self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: + return ScrapyPriorityQueue( + self.crawler, + self.downstream_queue_cls, + self.key + "/" + _path_safe(_scope_set_key(scope_set)), + startprios, + start_queue_cls=self._start_queue_cls, + ) + + def push(self, request: Request, scope_set: frozenset[ScopeID]) -> None: + now = time.monotonic() + self._promote_ready(now) + delay = self._throttler.get_request_delay(request, now) + if delay > 0: + self._delayed_seq += 1 + heapq.heappush( + self._delayed, (now + delay, self._delayed_seq, scope_set, request) + ) + return + self._push_to_queue(request, scope_set) + + def _push_to_queue(self, request: Request, scope_set: frozenset[ScopeID]) -> None: + if scope_set not in self.pqueues: + self.pqueues[scope_set] = self._pqfactory(scope_set) + self.pqueues[scope_set].push(request) + + def _promote_ready(self, now: float) -> None: + """Move every held-back request whose per-request delay has elapsed into + its scope-set queue, where it competes normally for its scopes.""" + while self._delayed and self._delayed[0][0] <= now: + self._release_delayed(heapq.heappop(self._delayed)) + + def _release_delayed( + self, entry: tuple[float, int, frozenset[ScopeID], Request] + ) -> None: + _, _, scope_set, request = entry + # The per-request delay has been honored (or the queue is closing), so + # mark it consumed: the request must not be delayed again, and on resume + # it must not re-block its scope set on a stale, no-longer-meaningful + # deadline. + request.meta["_throttler_delayed"] = True + try: + self._push_to_queue(request, scope_set) + except ValueError as e: + # A disk queue serializes on push; held-back requests defer that + # serialization until here, so a non-serializable one would + # otherwise raise while flushing on close and take the rest of the + # disk queue down with it. Drop it with a warning instead, matching + # how the scheduler handles unserializable requests at enqueue time. + logger.warning( + "Unable to serialize request: %(request)s - reason: %(reason)s", + {"request": request, "reason": e}, + exc_info=True, + extra={"spider": getattr(self.crawler, "spider", None)}, + ) + if self.crawler.stats is not None: + self.crawler.stats.inc_value("scheduler/unserializable") + + def _select( + self, + ) -> tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None: + """Return the sendable ``(scope_set, queue)`` pair to pop from, or + ``None`` if no queue can be popped from right now. + + Among the sendable queues (those whose scope set can be sent right now), + the one whose head has the highest request priority is chosen; ties are + broken by ascending load (the maximum + :meth:`~scrapy.throttler.ThrottlerProtocol.get_scope_load` over the + scopes of the queue), i.e. by preferring the least-busy scopes. + """ + self._promote_ready(time.monotonic()) + best_sort_key: tuple[int, float] | None = None + best: tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None = None + for scope_set, queue in self.pqueues.items(): + head = queue.peek() + if head is None or not self._throttler.is_ready(head): + continue + load = max( + (self._throttler.get_scope_load(scope_id) for scope_id in scope_set), + default=0.0, + ) + sort_key = (queue.priority(head), load) + if best_sort_key is None or sort_key < best_sort_key: + best_sort_key = sort_key + best = (scope_set, queue) + return best + + def pop(self) -> Request | None: + selected = self._select() + if selected is None: + return None + scope_set, queue = selected + request = queue.pop() + if request is not None: + self._throttler.reserve(request) + if len(queue) == 0: + del self.pqueues[scope_set] + return request + + def get_next_request_delay(self) -> float | None: + now = time.monotonic() + self._promote_ready(now) + delay: float | None = None + for queue in self.pqueues.values(): + head = queue.peek() + if head is None: + continue + if self._throttler.is_ready(head): + return 0.0 + head_delay = self._throttler.get_time_until_ready(head) + if head_delay is None: + continue + if delay is None or head_delay < delay: + delay = head_delay + # A request held back only by its own delay is not in any + # scope-set queue, so factor in when the earliest one is due. + if self._delayed: + next_delayed = max(0.0, self._delayed[0][0] - now) + if delay is None or next_delayed < delay: + delay = next_delayed + return delay + + def close(self) -> dict[str, list[int]]: + # Flush held-back requests into their scope-set queues so they are + # persisted (and restored on resume) rather than lost. + while self._delayed: + self._release_delayed(heapq.heappop(self._delayed)) + active = { + _scope_set_key(scope_set): queue.close() + for scope_set, queue in self.pqueues.items() + } + self.pqueues.clear() + return active + + def __len__(self) -> int: + queued = sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + return queued + len(self._delayed) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..dc3ba9426 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,16 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + :param user_agent: User agent + :type user_agent: str or bytes + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +95,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +135,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 932463ba9..cdf8b2f1b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ - if name == "CONCURRENT_REQUESTS_PER_IP" and ( - isinstance(self[name], int) and self[name] != 0 - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 049921ba7..b4ffa36f3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path -from typing import Any __all__ = [ "ADDONS", @@ -33,6 +32,14 @@ __all__ = [ "AWS_SESSION_TOKEN", "AWS_USE_SSL", "AWS_VERIFY", + "BACKOFF_DELAY_FACTOR", + "BACKOFF_ENABLED", + "BACKOFF_EXCEPTIONS", + "BACKOFF_HTTP_CODES", + "BACKOFF_JITTER", + "BACKOFF_MAX_DELAY", + "BACKOFF_MIN_DELAY", + "BACKOFF_WINDOW", "BOT_NAME", "CLOSESPIDER_ERRORCOUNT", "CLOSESPIDER_ITEMCOUNT", @@ -170,6 +177,7 @@ __all__ = [ "RANDOMIZE_DOWNLOAD_DELAY", "REACTOR_THREADPOOL_MAXSIZE", "REDIRECT_ENABLED", + "REDIRECT_MAX_DELAY", "REDIRECT_MAX_TIMES", "REDIRECT_PRIORITY_ADJUST", "REFERER_ENABLED", @@ -209,6 +217,16 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", + "THROTTLER", + "THROTTLER_DEBUG", + "THROTTLER_ROBOTSTXT_MAX_DELAY", + "THROTTLER_ROBOTSTXT_OBEY", + "THROTTLER_SCOPES", + "THROTTLER_SCOPE_CONCURRENCY", + "THROTTLER_SCOPE_LIMIT", + "THROTTLER_SCOPE_MANAGER", + "THROTTLER_SCOPE_MAX_IDLE", + "THROTTLER_WINDOW", "TWISTED_DNS_RESOLVER", "TWISTED_REACTOR", "TWISTED_REACTOR_ENABLED", @@ -235,6 +253,19 @@ AWS_SESSION_TOKEN = None AWS_USE_SSL = None AWS_VERIFY = None +BACKOFF_DELAY_FACTOR = 2.0 +BACKOFF_ENABLED = True +BACKOFF_EXCEPTIONS = [ + "scrapy.exceptions.DownloadFailedError", + "scrapy.exceptions.DownloadTimeoutError", + "scrapy.exceptions.ResponseDataLossError", +] +BACKOFF_HTTP_CODES = [429, 502, 503, 504, 520, 521, 522, 523, 524] +BACKOFF_JITTER = 0.1 +BACKOFF_MAX_DELAY = 300.0 +BACKOFF_MIN_DELAY = 1.0 +BACKOFF_WINDOW = 60.0 + BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 @@ -324,6 +355,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, + "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 650, "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, @@ -493,6 +525,7 @@ RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 REDIRECT_ENABLED = True +REDIRECT_MAX_DELAY = 120.0 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 @@ -574,9 +607,21 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +THROTTLER = "scrapy.throttler.Throttler" +THROTTLER_SCOPE_MANAGER = "scrapy.throttler.ThrottlerScopeManager" +THROTTLER_SCOPES = {} +THROTTLER_WINDOW = 60.0 +THROTTLER_ROBOTSTXT_OBEY = True +THROTTLER_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLER_SCOPE_CONCURRENCY = 1 +THROTTLER_SCOPE_LIMIT = 100000 +THROTTLER_SCOPE_MAX_IDLE = 3600.0 +THROTTLER_DEBUG = False + TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" TWISTED_REACTOR_ENABLED = True + TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" URLLENGTH_LIMIT = 2083 @@ -584,19 +629,3 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True - - -def __getattr__(name: str) -> Any: - if name == "CONCURRENT_REQUESTS_PER_IP": - import warnings # noqa: PLC0415 - - from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 - - warnings.warn( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return 0 - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0432a7231..aa3de5312 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -21,9 +21,8 @@ ADDONS = {} # Obey robots.txt rules ROBOTSTXT_OBEY = True -# Concurrency and throttling settings #CONCURRENT_REQUESTS = 16 -CONCURRENT_REQUESTS_PER_DOMAIN = 1 +THROTTLER_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 # Disable cookies (enabled by default) @@ -62,19 +61,6 @@ DOWNLOAD_DELAY = 1 # "$project_name.pipelines.${ProjectName}Pipeline": 300, #} -# Enable and configure the AutoThrottle extension (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/autothrottle.html -#AUTOTHROTTLE_ENABLED = True -# The initial download delay -#AUTOTHROTTLE_START_DELAY = 5 -# The maximum download delay to be set in case of high latencies -#AUTOTHROTTLE_MAX_DELAY = 60 -# The average number of requests Scrapy should be sending in parallel to -# each remote server -#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 -# Enable showing throttling stats for every response received: -#AUTOTHROTTLE_DEBUG = False - # Enable and configure HTTP caching (disabled by default) # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True diff --git a/scrapy/throttler.py b/scrapy/throttler.py new file mode 100644 index 000000000..171f550ce --- /dev/null +++ b/scrapy/throttler.py @@ -0,0 +1,1408 @@ +from __future__ import annotations + +import contextlib +import logging +import random +import time +import warnings +from collections import OrderedDict +from collections.abc import Awaitable, Callable, Iterable +from functools import wraps +from typing import TYPE_CHECKING, Any, Protocol, TypedDict, TypeVar, cast +from weakref import WeakKeyDictionary + +from twisted.internet.defer import Deferred +from typing_extensions import Self + +from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import SETTINGS_PRIORITIES +from scrapy.utils.asyncio import sleep, wait_for_first +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import build_from_crawler, load_object + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + from scrapy.http import Request + from scrapy.settings import BaseSettings + + +logger = logging.getLogger(__name__) + + +class BackoffConfig(TypedDict, total=False): + """Per-scope override of the backoff settings. + + Used as the value of the ``"backoff"`` key of :class:`ThrottlerScopeConfig` + entries. Any key left out falls back to the corresponding global + ``BACKOFF_*`` setting. + """ + + http_codes: list[int] + exceptions: list[str] + delay_factor: float + max_delay: float + min_delay: float + jitter: float | list[float] + + +class ThrottlerScopeConfig(TypedDict, total=False): + """Accepted keys of :setting:`THROTTLER_SCOPES` entries. + + Every key is optional; missing keys fall back to the matching global + setting (e.g. ``delay`` falls back to :setting:`DOWNLOAD_DELAY`). + """ + + concurrency: int + + delay: float + + jitter: float | list[float] + """Magnitude of the random variation applied to ``delay``; the per-scope + override of :setting:`RANDOMIZE_DOWNLOAD_DELAY` (``0`` disables it, ``0.5`` + means ±50%).""" + + quota: float + window: float + + manager: str | type + """Import path or class of a custom :setting:`THROTTLER_SCOPE_MANAGER` for + this scope.""" + + backoff: BackoffConfig + + ignore_robots_txt: bool + """Silence the warning logged when this configuration is more aggressive + than a robots.txt ``Crawl-delay``.""" + + +ScopeID = str +RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None] + + +def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: + """Iterate over the scope IDs of *scopes*, whatever its form. + + :class:`~ThrottlerProtocol.get_scopes` (and + :meth:`~ThrottlerProtocol.get_resolved_scopes`) may return a single + scope ID, an iterable of them, a ``{scope_id: quota}`` mapping, or ``None``; + this helper normalizes any of those into an iterable of scope IDs, e.g. to + react to a request's scopes in a custom middleware. + """ + return (scope for scope, _ in iter_scope_values(scopes)) + + +def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | None]]: + """Iterate over *scopes* as ``(scope_id, value)`` pairs. + + For dict scopes the value is the expected :ref:`throttler quota + ` consumption; for every other form the value is + ``None``. + """ + if scopes is None: + return + if isinstance(scopes, str): + yield scopes, None + return + if isinstance(scopes, dict): + yield from scopes.items() + return + for scope in scopes: + yield scope, None + + +def _effective_priority(settings: BaseSettings, name: str) -> int: + """Return the priority of setting *name*, treating an unset setting (no + priority, ``None``) as just below ``"default"`` so it never wins over one + that is at least at its default value.""" + priority = settings.getpriority(name) + return SETTINGS_PRIORITIES["default"] - 1 if priority is None else priority + + +def _default_scope_concurrency(settings: BaseSettings) -> int: + """Return the default concurrency of a throttler scope that does not set + its own ``concurrency``. + + This is :setting:`THROTTLER_SCOPE_CONCURRENCY`, except that the deprecated + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is bridged in when set at a + higher :ref:`priority `. When neither is set + explicitly, the historical :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` value is + kept for backward compatibility (its default flips to + :setting:`THROTTLER_SCOPE_CONCURRENCY` in a future version; see + :func:`_warn_on_deprecated_concurrency`). + """ + default_priority = SETTINGS_PRIORITIES["default"] + domain_priority = _effective_priority(settings, "CONCURRENT_REQUESTS_PER_DOMAIN") + scope_priority = _effective_priority(settings, "THROTTLER_SCOPE_CONCURRENCY") + if domain_priority > scope_priority: + return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + if scope_priority > domain_priority: + return settings.getint("THROTTLER_SCOPE_CONCURRENCY") + # Equal priority: on an explicit (higher-than-default) tie the new setting + # wins; when neither is set (both at "default") keep the historical + # per-domain value so existing behavior is preserved. + if domain_priority <= default_priority: + return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + return settings.getint("THROTTLER_SCOPE_CONCURRENCY") + + +def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: + """Warn about the concurrency settings bridged by + :func:`_default_scope_concurrency`. Call once per crawl (see + :meth:`Throttler.__init__`). + + When :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is set explicitly, warn that + it is deprecated. When neither concurrency setting is set explicitly, warn + that the effective per-scope concurrency is still the deprecated setting's + value (kept for backward compatibility) and will drop to + :setting:`THROTTLER_SCOPE_CONCURRENCY`'s default once the deprecated + setting is removed, so users can pin it explicitly.""" + default_priority = SETTINGS_PRIORITIES["default"] + domain_set = ( + _effective_priority(settings, "CONCURRENT_REQUESTS_PER_DOMAIN") + > default_priority + ) + scope_set = ( + _effective_priority(settings, "THROTTLER_SCOPE_CONCURRENCY") > default_priority + ) + if domain_set: + warnings.warn( + "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " + "THROTTLER_SCOPE_CONCURRENCY instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + elif not scope_set: + # This warn-then-flip message only makes sense while the two defaults + # differ (otherwise it reads "will drop from 1 to 1"). That invariant is + # guarded by test_deprecated_concurrency_defaults_differ rather than at + # run time, so a crawl is never aborted over it. + current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + future = settings.getint("THROTTLER_SCOPE_CONCURRENCY") + warnings.warn( + f"The effective per-scope (per-domain) concurrency is {current}, " + f"the default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN " + f"setting, which is still respected for backward compatibility. " + f"Once CONCURRENT_REQUESTS_PER_DOMAIN is removed, it will drop to " + f"{future}, the default of THROTTLER_SCOPE_CONCURRENCY. Set " + f"THROTTLER_SCOPE_CONCURRENCY explicitly to choose a value and " + f"silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + +def _warn_on_unachievable_concurrency(settings: BaseSettings) -> None: + """Warn about configured concurrency limits that exceed + :setting:`CONCURRENT_REQUESTS`. Call once per crawl (see + :meth:`Throttler.__init__`). + + :setting:`CONCURRENT_REQUESTS` caps the total number of requests in flight, + so a per-scope (or per-domain) concurrency limit above it can never be + reached. + """ + global_concurrency = settings.getint("CONCURRENT_REQUESTS") + offenders: list[str] = [ + f"{name}={settings.getint(name)}" + for name in ("CONCURRENT_REQUESTS_PER_DOMAIN", "THROTTLER_SCOPE_CONCURRENCY") + if settings.getint(name) > global_concurrency + ] + offenders += [ + f"THROTTLER_SCOPES[{scope_id!r}]['concurrency']={config['concurrency']}" + for scope_id, config in settings.getdict("THROTTLER_SCOPES").items() + if config.get("concurrency") is not None + and int(config["concurrency"]) > global_concurrency + ] + if offenders: + logger.warning( + f"The following concurrency settings exceed CONCURRENT_REQUESTS " + f"({global_concurrency}), which caps the total number of requests in " + f"flight, so they cannot be reached: {', '.join(offenders)}." + ) + + +def _to_scope_dict(scopes: RequestScopes) -> dict[ScopeID, float | None]: + """Normalize *scopes* (``None``, a scope id, an iterable of scope ids or a + ``{scope_id: quota}`` dict) into a ``{scope_id: quota}`` dict, using ``None`` + as the quota of scopes that have none.""" + if isinstance(scopes, dict): + return scopes + if scopes is None: + return {} + if isinstance(scopes, str): + return {scopes: None} + if isinstance(scopes, Iterable): + return dict.fromkeys(scopes) + raise TypeError( + f"Invalid type ({type(scopes)}) of scopes value " + f"{scopes!r}. Expected None, str, Iterable or dict." + ) + + +def add_scope( + scopes: RequestScopes, + scope: ScopeID, + value: float | None = None, + /, +) -> dict[ScopeID, float | None]: + """Add *scope* to *scopes* with *value*, returning a ``{scope_id: quota}`` + dict. + + This is a utility function to help extending the output of + :meth:`~ThrottlerProtocol.get_scopes`, e.g. in + :class:`Throttler` subclasses. + + Adding a scope with a *value* fails if it is already present, so an existing + :ref:`quota ` is never silently overwritten; adding it + without a value leaves any existing entry untouched. + """ + result = _to_scope_dict(scopes) + if value is None: + result.setdefault(scope, None) + return result + if scope in result: + raise TypeError(f"Scope {scope!r} already has a value in {scopes!r}") + result[scope] = value + return result + + +class ThrottlerProtocol(Protocol): + """A protocol for :setting:`THROTTLER` :ref:`components + `.""" + + async def get_scopes(self, request: Request) -> RequestScopes: + """Return the :ref:`throttler scopes ` that apply + to *request*. + + Return ``None`` if no scopes apply, a string for a single scope, an + iterable of strings for multiple scopes, or a dict with scope names as + keys and :ref:`throttler quotas ` as values. + """ + + def get_resolved_scopes(self, request: Request) -> RequestScopes: + """Return the :ref:`throttler scopes ` under which + *request* was (or will be) sent, without re-resolving them. + + This is the synchronous counterpart of :meth:`get_scopes`: it returns + the scopes resolved earlier (e.g. at enqueue or :meth:`acquire` time) + and persisted on ``request.meta``, falling back to a best-effort + synchronous resolution only if none were persisted. Use it, rather than + :meth:`get_scopes`, to attribute a response or exception to the very + scopes the request was sent under — e.g. from a downloader middleware or + a spider callback that wants to :meth:`back_off` based on the response. + """ + + async def acquire(self, request: Request) -> None: + """Block until *request* is allowed to be sent by all of its scopes. + + This is the throttling gate that the engine awaits before releasing a + request to the downloader. + """ + + def release(self, request: Request) -> None: + """Release the concurrency slots that :meth:`acquire` reserved for + *request*. + + The engine calls this once *request* has finished downloading (whether + it succeeded, failed or returned a new request), so that scopes that + enforce a concurrency limit can let other requests through. + """ + + def is_ready(self, request: Request) -> bool: + """Return whether every scope of *request* allows it to be sent right + now, i.e. all time-based gates (delay, backoff, quota window) are open + *and* a concurrency slot is free in every scope. + + This is the synchronous, non-blocking counterpart of :meth:`acquire`, + used by a :ref:`throttler-aware scheduler + ` to decide whether a request can be + dequeued now. It assumes the scopes of *request* have already been + resolved (e.g. by an earlier :meth:`get_scopes` call at enqueue time). + """ + + def reserve(self, request: Request) -> None: + """Claim a send for *request*: record the send on every one of its + scopes and mark *request* as reserved, so that a later :meth:`acquire` + for it returns immediately without reserving again. + + A :ref:`throttler-aware scheduler ` calls + this when it decides to dequeue *request* (after :meth:`is_ready` + returned ``True``). The reservation is released by :meth:`release`. + """ + + def get_time_until_ready(self, request: Request) -> float | None: + """Return the number of seconds until every time-based gate of + *request* would be open, or ``None`` if no time-based gate is currently + blocking it (only a concurrency slot could be). + + Used by a :ref:`throttler-aware scheduler + ` to schedule a wakeup when all pending + requests are time-blocked. + """ + + def get_slot_key(self, request: Request) -> str: + """Return a single string key for *request*, derived from its scopes. + + For a single scope this is the scope ID itself; for multiple scopes + the sorted scope IDs are joined with ``"+"``. This is the synchronous + counterpart of :meth:`get_scopes`, used wherever a plain string key is + needed (e.g. scheduler priority queues). + """ + + def get_scope_load(self, scope_id: str) -> float: + """Return the current load of the scope identified by *scope_id*: its + active sends divided by its concurrency limit (or by the global + :setting:`CONCURRENT_REQUESTS` when the scope has no explicit limit). + + Used by a :ref:`throttler-aware scheduler + ` to balance dequeuing across scopes, + preferring the least-loaded ones. + """ + + def get_request_delay(self, request: Request, now: float | None = None) -> float: + """Return how many seconds *request* must still be held individually + because of its :reqmeta:`delay`, or ``0.0`` if it has none + or it has already elapsed. The one-time delay is started on the first + call. + + Unlike a scope delay, this affects only *request*: a + :ref:`throttler-aware scheduler ` must + hold the request back on its own, **without** blocking other requests + that share its scopes. + """ + + def back_off( + self, + scopes: RequestScopes, + *, + delay: float | None = None, + cap: bool = True, + ) -> None: + """Register a :ref:`backoff ` trigger for each of *scopes*. + + This is the general-purpose way to make a scope slow down, available to + any component through :attr:`crawler.throttler + `. The built-in :class:`backoff + middleware ` + calls it for :setting:`BACKOFF_HTTP_CODES` responses and + :setting:`BACKOFF_EXCEPTIONS` exceptions, but a downloader middleware or + spider callback can call it too (e.g. to back off based on the response + body of a specific site). + + *scopes* accepts the same shapes as the output of :meth:`get_scopes` + (typically the result of :meth:`get_resolved_scopes` for a request). + + An exponential backoff step is always applied to the scope's delay. + When *delay* is given, the scope is *additionally* held back for at + least *delay* seconds before its next request: a one-time gate (e.g. + from a :ref:`Retry-After ` header), not a change to the + steady-state delay. *cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`; + set it to ``False`` for trusted, programmatic delays. + """ + + def delay_scope(self, scope_id: str, delay: float) -> None: + """Hold back the scope identified by *scope_id* for at least *delay* + seconds before its next request, and register a :ref:`backoff + ` trigger for the scope. + + Like a :ref:`Retry-After ` response header, this is a + one-time delay (the scope's steady-state delay grows by one backoff + step and then recovers), not a permanent one; call it again to keep a + scope slowed down for longer. + + This is shorthand for :meth:`back_off(scope_id, delay=delay, + cap=False) `. Unlike a ``Retry-After`` header, *delay* is + **not** capped at :setting:`BACKOFF_MAX_DELAY`: that cap guards against + untrusted input, whereas a ``delay_scope`` call is trusted. + """ + + def reconcile_quota( + self, + scopes: RequestScopes, + *, + consumed: float | None = None, + remaining: float | None = None, + ) -> None: + """Reconcile the :ref:`throttler quota ` of each of + *scopes* with an actually *consumed* amount (a delta to add) or a + *remaining* amount (an absolute value), correcting the estimate used + when requests were sent. + + Like :meth:`back_off`, this is meant to be called from a downloader + middleware or spider callback that learns the real quota cost of a + request from its response. + """ + + def get_scope_delay(self, scope_id: str) -> float: + """Return the current base (non-backoff) delay of *scope_id*, in seconds.""" + + def set_scope_delay(self, scope_id: str, delay: float) -> None: + """Set the base (non-backoff) delay of *scope_id* to *delay* seconds. + + Unlike :meth:`delay_scope`, this both raises and lowers the delay and is + not counted as backoff; it lets a component drive the scope delay + directly (e.g. an adaptive-delay extension). + """ + + def get_scope_manager(self, scope_id: str) -> ThrottlerScopeManagerProtocol: + """Return the :class:`ThrottlerScopeManagerProtocol` instance handling + the scope identified by *scope_id*, creating it if necessary.""" + + +_GetScopesMethod = TypeVar( + "_GetScopesMethod", bound=Callable[..., Awaitable[RequestScopes]] +) + + +# Request.meta key under which scope_cache persists the resolved scopes so that +# they survive a request being serialized to and restored from a disk queue. +_RESOLVED_SCOPES_META_KEY = "_throttler_resolved_scopes" + + +def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: + """Decorator for :meth:`~ThrottlerProtocol.get_scopes` + implementations that persists the resolved scopes on ``request.meta``. + + The readers of the resolved scopes — the synchronous readiness API of a + :ref:`throttler-aware scheduler ` and + :meth:`~ThrottlerProtocol.get_resolved_scopes` — read this persisted + value instead of resolving the scopes again, so they stay cheap and + consistent, and it survives a request being + serialized to and restored from a :ref:`disk queue ` (which is + what lets the readiness API resolve the scopes of a restored request + synchronously). + + The decorated method always re-resolves; it never reads the persisted value + back. So a request that inherited ``request.meta`` from another one (e.g. a + redirect built with :meth:`Request.replace() `, + which copies ``meta``) resolves its own scopes and overwrites the inherited + ones rather than reusing them. + + For example: + + .. code-block:: python + + from scrapy.utils.httpobj import urlparse_cached + from scrapy.throttler import scope_cache + + + class MyThrottler: + @scope_cache + async def get_scopes(self, request): + return urlparse_cached(request).hostname or "" + """ + + @wraps(f) + async def wrapper(self: Any, request: Request) -> RequestScopes: + scopes = await f(self, request) + # Materialize one-shot iterables so the persisted value stays + # re-iterable and serializable. + if not isinstance(scopes, (str, dict)) and isinstance(scopes, Iterable): + scopes = list(scopes) + request.meta[_RESOLVED_SCOPES_META_KEY] = scopes + return scopes + + return wrapper # type: ignore[return-value] + + +class Throttler: + """The default :setting:`THROTTLER` class. + + It assigns to each request its domain or subdomain as scope and handles + backoff according to :ref:`backoff settings `. + """ + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def __init__(self, crawler: Crawler) -> None: + self.crawler = crawler + _warn_on_deprecated_concurrency(crawler.settings) + _warn_on_unachievable_concurrency(crawler.settings) + self._debug = crawler.settings.getbool("THROTTLER_DEBUG") + self._max_idle = crawler.settings.getfloat("THROTTLER_SCOPE_MAX_IDLE") + self._robotstxt_obey = crawler.settings.getbool( + "ROBOTSTXT_OBEY" + ) and crawler.settings.getbool("THROTTLER_ROBOTSTXT_OBEY") + self._robotstxt_max_delay = crawler.settings.getfloat( + "THROTTLER_ROBOTSTXT_MAX_DELAY" + ) + self._default_useragent: str = crawler.settings["USER_AGENT"] + self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] + if self._robotstxt_obey: + crawler.signals.connect( + self._on_robots_parsed, signal=signals.robots_parsed + ) + self._default_scope_manager_cls = load_object( + crawler.settings["THROTTLER_SCOPE_MANAGER"] + ) + self._scopes_config: dict[str, dict[str, Any]] = self._merge_download_slots( + crawler.settings + ) + # Ordered by least-recently-used first (see get_scope_manager), so the + # scope limit can evict the coldest idle scopes (see THROTTLER_SCOPE_LIMIT). + self._scope_managers: OrderedDict[ScopeID, ThrottlerScopeManagerProtocol] = ( + OrderedDict() + ) + self._scope_limit: int = crawler.settings.getint("THROTTLER_SCOPE_LIMIT") + self._last_eviction: float | None = None + # Concurrency slots reserved by acquire(), to be released once the + # request finishes downloading. + self._reserved: WeakKeyDictionary[ + Request, list[tuple[ThrottlerScopeManagerProtocol, float | None]] + ] = WeakKeyDictionary() + + @staticmethod + def _merge_download_slots(settings: BaseSettings) -> dict[str, dict[str, Any]]: + """Return the effective per-scope configuration, merging the deprecated + :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLER_SCOPES`. + + Each ``DOWNLOAD_SLOTS`` entry is translated to a throttler scope keyed + by the same slot name (the default manager keys domain scopes by host + name, which is what download slots used too): ``concurrency`` and + ``delay`` map directly, and the ``randomize_delay`` boolean maps to a + ``jitter`` magnitude (the historical ±50%, or none). An explicit + ``THROTTLER_SCOPES`` entry for the same scope takes precedence over the + translated one. The deprecation warning is emitted by the downloader. + """ + scopes: dict[str, dict[str, Any]] = { + scope_id: dict(config) + for scope_id, config in settings.getdict("THROTTLER_SCOPES").items() + } + for slot_id, slot_config in settings.getdict("DOWNLOAD_SLOTS").items(): + translated: dict[str, Any] = {} + if "concurrency" in slot_config: + translated["concurrency"] = slot_config["concurrency"] + if "delay" in slot_config: + translated["delay"] = slot_config["delay"] + if "randomize_delay" in slot_config: + translated["jitter"] = 0.5 if slot_config["randomize_delay"] else 0.0 + # An explicit THROTTLER_SCOPES entry wins over the translated one. + scopes[slot_id] = {**translated, **scopes.get(slot_id, {})} + return scopes + + @scope_cache + async def get_scopes(self, request: Request) -> RequestScopes: + return self._resolve_scopes_sync(request) + + def _resolve_scopes_sync(self, request: Request) -> RequestScopes: + """Best-effort synchronous scope resolution. + + It backs :meth:`get_scopes` and is also the fallback for the synchronous + readiness methods (:meth:`is_ready`, :meth:`reserve`, + :meth:`get_time_until_ready`) when no scopes were persisted on + ``request.meta`` by an earlier :meth:`get_scopes` call (which normally + happens at enqueue time and survives disk restores; see + :func:`scope_cache`). Subclasses whose :meth:`get_scopes` cannot be + resolved synchronously rely on that persisted value instead. + """ + scopes = request.meta.get("throttler_scopes") + if scopes is not None: + return cast("RequestScopes", scopes) + download_slot = request.meta.get("download_slot") + if download_slot is not None: + warnings.warn( + "The 'download_slot' request meta key is deprecated. Use " + "'throttler_scopes' instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return cast("RequestScopes", download_slot) + return urlparse_cached(request).hostname or "" + + def get_slot_key(self, request: Request) -> str: + scopes = self._resolve_scopes_sync(request) + scope_ids = sorted(iter_scopes(scopes)) + return "+".join(scope_ids) if scope_ids else "" + + def get_resolved_scopes(self, request: Request) -> RequestScopes: + """Return the scopes under which *request* was (or will be) sent, + reusing those persisted on ``request.meta`` by an earlier + :meth:`get_scopes` call (see :func:`scope_cache`) and falling back to + :meth:`_resolve_scopes_sync` only when none were persisted. + + The persisted value is authoritative: a request reaching a reader of + this method (the readiness API, or a component reacting to its response) + has already been enqueued and, for a request restored from a disk queue, + re-resolving could attribute it to different scopes than the ones it was + sent under. + """ + if _RESOLVED_SCOPES_META_KEY in request.meta: + return cast("RequestScopes", request.meta[_RESOLVED_SCOPES_META_KEY]) + return self._resolve_scopes_sync(request) + + def _cached_scope_values( + self, request: Request + ) -> list[tuple[ScopeID, float | None]]: + """Return the ``(scope_id, quota_amount)`` pairs of *request*, from the + scopes returned by :meth:`get_resolved_scopes`.""" + return list(iter_scope_values(self.get_resolved_scopes(request))) + + # -- Scope-state coordination (called from the request lifecycle) -------- + + def get_scope_manager(self, scope_id: ScopeID) -> ThrottlerScopeManagerProtocol: + manager = self._scope_managers.get(scope_id) + if manager is not None: + # Mark as most-recently-used for the LRU scope limit. + self._scope_managers.move_to_end(scope_id) + return manager + config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {})) + config.setdefault("id", scope_id) + manager_cls = ( + load_object(config["manager"]) + if "manager" in config + else self._default_scope_manager_cls + ) + manager = cast( + "ThrottlerScopeManagerProtocol", + build_from_crawler(manager_cls, self.crawler, config), + ) + self._scope_managers[scope_id] = manager + self._enforce_scope_limit(scope_id) + return manager + + def _enforce_scope_limit(self, keep: ScopeID) -> None: + """Evict least-recently-used idle scopes while the number of live scope + managers exceeds :setting:`THROTTLER_SCOPE_LIMIT` (``0`` disables the + limit). + + LRU order is kept by :meth:`get_scope_manager` moving each accessed + scope to the end, so the coldest scopes are at the front. Only scopes + that are idle (no in-flight requests and no active backoff) are evicted; + the just-created *keep* scope is never evicted. A scope evicted while + still throttling is simply recreated from its configuration the next + time it is needed. + """ + if self._scope_limit <= 0 or len(self._scope_managers) <= self._scope_limit: + return + now = time.monotonic() + for scope_id in list(self._scope_managers): + if len(self._scope_managers) <= self._scope_limit: + break + if scope_id != keep and self._scope_managers[scope_id].is_idle(now, 0): + del self._scope_managers[scope_id] + + async def acquire(self, request: Request) -> None: + # A throttler-aware scheduler reserves the request before handing it + # to the engine, so there is nothing left to wait for or record here. + if request in self._reserved: + return + now = time.monotonic() + self._maybe_evict(now) + await self._delay_request(request) + scope_values = list(iter_scope_values(await self.get_scopes(request))) + if not scope_values: + return + managers = [ + (self.get_scope_manager(scope_id), value) + for scope_id, value in scope_values + ] + while True: + wait = max( + [0.0, *(manager.can_send(amount=value) for manager, value in managers)] + ) + if wait > 0: + if self._debug: + logger.debug( + f"Throttling {request} for {wait:.2f}s " + f"(scopes: {[scope_id for scope_id, _ in scope_values]})" + ) + await sleep(wait) + continue + # All time-based gates (delay, backoff, quota) are open; the only + # remaining reason to wait is a full concurrency slot. + blocked = [ + manager for manager, _ in managers if manager.concurrency_blocked() + ] + if not blocked: + self._record_reservation(request, managers) + return + if self._debug: + logger.debug( + f"Throttling {request} until a concurrency slot frees up " + f"(scopes: {[scope_id for scope_id, _ in scope_values]})" + ) + await self._wait_for_slot(blocked) + + def _record_reservation( + self, + request: Request, + managers: list[tuple[ThrottlerScopeManagerProtocol, float | None]], + ) -> None: + """Record a send on each of *request*'s scope *managers* and mark + *request* as reserved, so :meth:`release` can later free the slots. This + is the shared tail of :meth:`acquire` and :meth:`reserve`.""" + for manager, value in managers: + manager.record_sent(amount=value) + self._reserved[request] = managers + + def release(self, request: Request) -> None: + managers = self._reserved.pop(request, None) + if not managers: + return + for manager, _ in managers: + manager.record_done() + + # -- Synchronous readiness API (used by a throttler-aware scheduler) ------ + + def is_ready(self, request: Request) -> bool: + now = time.monotonic() + if self._request_delay_deadline(request, now) > now: + return False + for scope_id, value in self._cached_scope_values(request): + manager = self.get_scope_manager(scope_id) + if manager.can_send(now=now, amount=value) > 0: + return False + if manager.concurrency_blocked(): + return False + return True + + def reserve(self, request: Request) -> None: + # A throttler-aware scheduler reserves every request before handing it + # to the engine, so acquire() always returns early for it and never + # gets to evict idle scopes; do it here so their managers do not pile + # up on broad crawls. + self._maybe_evict(time.monotonic()) + managers = [ + (self.get_scope_manager(scope_id), value) + for scope_id, value in self._cached_scope_values(request) + ] + self._record_reservation(request, managers) + + def get_time_until_ready(self, request: Request) -> float | None: + now = time.monotonic() + wait = max(0.0, self._request_delay_deadline(request, now) - now) + for scope_id, value in self._cached_scope_values(request): + manager = self.get_scope_manager(scope_id) + wait = max(wait, manager.can_send(now=now, amount=value)) + return wait if wait > 0 else None + + def get_scope_load(self, scope_id: ScopeID) -> float: + return self.get_scope_manager(scope_id).get_load() + + def get_request_delay(self, request: Request, now: float | None = None) -> float: + now = time.monotonic() if now is None else now + return max(0.0, self._request_delay_deadline(request, now) - now) + + async def _wait_for_slot(self, managers: list[Any]) -> None: + """Block until any of *managers* frees a concurrency slot. + + Each manager hands out an event Deferred that fires when a slot is freed + (via :meth:`ThrottlerScopeManager.record_done`) or the limit is raised + (via :meth:`ThrottlerScopeManager.set_concurrency`). A long safety timer + bounds the wait in case no slot is ever freed (it always should be, via + :meth:`release`). + """ + pairs = [(manager, manager.slot_event()) for manager in managers] + events = [event for _, event in pairs] + _, pending = await wait_for_first(events, timeout=_SLOT_WAIT_TIMEOUT) + for manager, event in pairs: + if event in pending: + manager.discard_slot_event(event) + + async def _delay_request(self, request: Request) -> None: + """Honor the :reqmeta:`delay` meta key by holding *request* + for the requested number of seconds the first time it is processed. + + This is the blocking (:meth:`acquire`) counterpart of + :meth:`_request_delay_deadline`, which the readiness API polls instead; + both share the deadline bookkeeping and the one-time debug log. Here the + deadline is honored by sleeping until it, then marking the delay as + consumed so the request is never held again.""" + now = time.monotonic() + wait = self._request_delay_deadline(request, now) - now + if wait <= 0: + return + await sleep(wait) + request.meta["_throttler_delayed"] = True + + def _request_delay_deadline(self, request: Request, now: float) -> float: + """Return the monotonic time before which *request* must not be sent due + to its :reqmeta:`delay`, or ``0.0`` if it has none. + + This is the readiness-API counterpart of :meth:`_delay_request`: + a throttler-aware scheduler gates requests through :meth:`is_ready` and + :meth:`get_time_until_ready` instead of awaiting :meth:`acquire`, so the + delay is enforced by holding back the request until this deadline rather + than by sleeping. The deadline is computed once, the first time the + request reaches the gate, and stored so later polls reuse it. + + A request whose delay has already been honored (the ``_throttler_delayed`` + flag, also set by :meth:`_delay_request`) is never delayed again, + which keeps a resumed crawl from re-blocking on a stale deadline.""" + delay = request.meta.get("delay") + if not delay or request.meta.get("_throttler_delayed"): + return 0.0 + deadline = request.meta.get("_throttler_delay_deadline") + if deadline is None: + deadline = now + float(delay) + request.meta["_throttler_delay_deadline"] = deadline + if self._debug: + logger.debug(f"Holding {request} for {delay:.2f}s (delay)") + return deadline + + def back_off( + self, + scopes: RequestScopes, + *, + delay: float | None = None, + cap: bool = True, + ) -> None: + for scope_id in iter_scopes(scopes): + if self._debug: + logger.debug(f"Backoff for scope {scope_id} (delay: {delay})") + self.get_scope_manager(scope_id).record_backoff(delay=delay, cap=cap) + + def reconcile_quota( + self, + scopes: RequestScopes, + *, + consumed: float | None = None, + remaining: float | None = None, + ) -> None: + for scope_id in iter_scopes(scopes): + self.get_scope_manager(scope_id).reconcile_quota( + consumed=consumed, remaining=remaining + ) + + def _on_robots_parsed(self, robotparser: Any, request: Request) -> None: + """Honor a robots.txt ``Crawl-delay`` on the :signal:`robots_parsed` + signal. + + It reads the ``Crawl-delay`` directive for the configured user agent from + the parsed robots.txt and, if present, applies it to the scope of the + host that *request* targets via :meth:`apply_robots_crawl_delay`. + """ + if not self._robotstxt_obey: + return + useragent: str | bytes = self._robotstxt_useragent or self._default_useragent + try: + delay = robotparser.crawl_delay(useragent) + except Exception: # pragma: no cover - backend-specific failures + return + if delay: + self.apply_robots_crawl_delay( + urlparse_cached(request).hostname or "", delay + ) + + def apply_robots_crawl_delay(self, scope_id: ScopeID, delay: float) -> None: + """Honor a robots.txt ``Crawl-delay`` directive of *delay* seconds for + *scope_id* by setting its delay (capped at + :setting:`THROTTLER_ROBOTSTXT_MAX_DELAY`) and its concurrency to ``1``. + + Called from the :signal:`robots_parsed` signal handler when + :setting:`THROTTLER_ROBOTSTXT_OBEY` is enabled. An explicit + :setting:`THROTTLER_SCOPES` configuration for the scope is respected, but + a warning is logged about the discrepancy unless its ``ignore_robots_txt`` + key is ``True``. + """ + if not self._robotstxt_obey: + return + capped = min(delay, self._robotstxt_max_delay) + config = self._scopes_config.get(scope_id, {}) + if config.get("ignore_robots_txt"): + return + conflicts = [] + if config.get("delay") is not None and float(config["delay"]) < capped: + conflicts.append(f"delay={config['delay']!r} < Crawl-delay {capped}") + if config.get("concurrency") is not None and int(config["concurrency"]) > 1: + conflicts.append(f"concurrency={config['concurrency']!r} > 1") + if conflicts: + logger.warning( + f"Throttler scope {scope_id!r} is configured with {' and '.join(conflicts)}, " + f"which is more aggressive than its robots.txt Crawl-delay of " + f"{capped}s. The configured values take precedence; set " + "'ignore_robots_txt': True in its THROTTLER_SCOPES entry to " + "silence this warning." + ) + return + if self._debug: + logger.debug(f"robots.txt Crawl-delay for scope {scope_id}: {capped}s") + manager = self.get_scope_manager(scope_id) + manager.set_base_delay(capped) + manager.set_concurrency(1) + + def delay_scope(self, scope_id: ScopeID, delay: float) -> None: + # Like a Retry-After / RateLimit-Reset header, this gates the scope's + # next request by delay seconds (a one-time hold, not a steady-state + # delay); unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. + self.back_off(scope_id, delay=float(delay), cap=False) + + def get_scope_delay(self, scope_id: ScopeID) -> float: + return self.get_scope_manager(scope_id).get_base_delay() + + def set_scope_delay(self, scope_id: ScopeID, delay: float) -> None: + self.get_scope_manager(scope_id).set_base_delay( + float(delay), only_increase=False + ) + + def _maybe_evict(self, now: float) -> None: + if self._max_idle <= 0: + return + if ( + self._last_eviction is not None + and now - self._last_eviction < self._max_idle / 2 + ): + return + self._last_eviction = now + for scope_id in list(self._scope_managers): + if self._scope_managers[scope_id].is_idle(now, self._max_idle): + del self._scope_managers[scope_id] + + +class ThrottlerScopeManagerProtocol(Protocol): + """A protocol for :setting:`THROTTLER_SCOPE_MANAGER` :ref:`components + `. + + The ``__init__`` method gets a ``config`` dict with the base configuration + of the managed throttler scope. For example: + + .. code-block:: python + + { + "id": "example.com", + "concurrency": 1, + "delay": 1.0, + "jitter": 0.5, + "quota": 1000.0, + "window": 60.0, + "backoff": { + "http_codes": [429, 503], + "exceptions": ["builtins.IOError"], + "delay_factor": 1.2, + "max_delay": 180.0, + "min_delay": 5.0, + "jitter": [0.01, 0.33], + }, + } + + """ + + @classmethod + def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self: ... + + def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: ... + + def can_send(self, now: float | None = None, amount: float | None = None) -> float: + """Return the number of seconds to wait before a request for this scope + may be sent, or ``0`` if it may be sent right away. + + *amount* is the expected :ref:`throttler quota ` + consumption of the request, if any. + """ + + def record_sent( + self, now: float | None = None, amount: float | None = None + ) -> None: + """Record that a request for this scope has just been sent, consuming + *amount* of its :ref:`throttler quota ` if given.""" + + def record_done(self, now: float | None = None) -> None: + """Record that a previously :meth:`record_sent` request has finished + downloading, freeing its concurrency slot.""" + + def record_backoff( + self, + delay: float | None = None, + now: float | None = None, + cap: bool = True, + ) -> None: + """Apply a backoff to this scope. + + *delay*, when given, is a hard minimum delay in seconds (e.g. from a + ``Retry-After`` header). When omitted, an exponential backoff step is + applied instead. + + *cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`. It is ``True`` for + untrusted input such as response headers, and may be set to ``False`` + for trusted, programmatic delays (see + :meth:`ThrottlerProtocol.delay_scope`). + """ + + def reconcile_quota( + self, + consumed: float | None = None, + remaining: float | None = None, + now: float | None = None, + ) -> None: + """Reconcile the :ref:`throttler quota ` of this + scope with the actual *consumed* amount (or the *remaining* amount) + reported for a request, correcting the estimate used by + :meth:`record_sent`.""" + + def get_base_delay(self) -> float: + """Return the base (non-backoff) delay of this scope, in seconds.""" + + def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None: + """Set the base (non-backoff) delay of this scope to *delay* seconds. + + By default it only raises the delay, to honor external hints such as a + robots.txt ``Crawl-delay`` directive. Pass ``only_increase=False`` to + also allow lowering it. + """ + + def set_concurrency(self, concurrency: int) -> None: + """Set the maximum number of concurrent requests allowed for this + scope.""" + + def concurrency_blocked(self) -> bool: + """Return whether this scope is at its concurrency limit. + + :class:`Throttler` calls this (after all time-based gates in + :meth:`can_send` are open) to decide whether to wait for a freed slot. + Return ``False`` when no concurrency limit is enforced. + """ + + def get_load(self) -> float: + """Return the current load of this scope: a non-negative number, with + ``1.0`` meaning "as busy as its concurrency limit allows". + + A :ref:`throttler-aware scheduler ` uses + this to break ties between equally-prioritized requests, preferring the + least-loaded scopes. The reference implementation returns active sends + divided by the concurrency limit (falling back to + :setting:`CONCURRENT_REQUESTS` when the scope enforces no explicit + limit), but any consistent busyness metric works; return ``0.0`` when + none is meaningful. + """ + + def slot_event(self) -> Deferred[None]: + """Return a :class:`~twisted.internet.defer.Deferred` that fires when + a concurrency slot next becomes available (e.g. when + :meth:`record_done` is called or the limit is raised via + :meth:`set_concurrency`).""" + + def discard_slot_event(self, event: Deferred[None]) -> None: + """Cancel a pending slot event returned by :meth:`slot_event`. + + Called by :class:`Throttler` when the wait ends without the + event firing (e.g. another scope's slot opened first). + """ + + def is_idle(self, now: float, max_idle: float) -> bool: + """Return whether this scope can be evicted from memory. + + A scope is idle when it has not been used for *max_idle* seconds and is + not currently in an active (future) backoff. + """ + + +# Safety timeout for acquire() while it waits, event-driven, for a concurrency +# slot to free up: a slot_event normally fires first (from record_done() or +# set_concurrency()); this only guards against a request that never reaches +# release() so the wait can never hang forever. +_SLOT_WAIT_TIMEOUT = 1.0 + + +class ThrottlerScopeManager: + """The default :setting:`THROTTLER_SCOPE_MANAGER` class. + + It implements a per-scope state machine covering delay, exponential + :ref:`backoff `, concurrency and :ref:`quotas + `: + + - A base delay (the scope ``"delay"`` config, defaulting to + :setting:`DOWNLOAD_DELAY`) is enforced between consecutive requests for + the scope. + + - On a backoff trigger (a :setting:`BACKOFF_HTTP_CODES` response or a + :setting:`BACKOFF_EXCEPTIONS` exception) the delay grows exponentially + by :setting:`BACKOFF_DELAY_FACTOR`, bounded by :setting:`BACKOFF_MIN_DELAY` + and :setting:`BACKOFF_MAX_DELAY`, with :setting:`BACKOFF_JITTER` applied. + A ``Retry-After`` / ``RateLimit-Reset`` delay additionally holds the + scope back until that time before its next request (a one-time gate, + capped at :setting:`BACKOFF_MAX_DELAY`), without becoming the + steady-state delay. + + - After :setting:`BACKOFF_WINDOW` seconds without a new trigger, the delay + recovers one step at a time back towards the base delay. + + - When the scope is configured with a ``"concurrency"`` limit, no more + than that many requests are allowed in flight at once. + + - When the scope is configured with a ``"quota"``, no more than that much + quota is consumed per ``"window"`` (default: :setting:`THROTTLER_WINDOW`). + """ + + @classmethod + def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self: + return cls(crawler, config) + + def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: + settings = crawler.settings + backoff: dict[str, Any] = config.get("backoff", {}) + self._id: ScopeID = config.get("id", "") + # The per-scope delay defaults to DOWNLOAD_DELAY; a scope can override + # it with its own "delay" config (see THROTTLER_SCOPES). + self._base_delay: float = float( + config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) + ) + # Magnitude of the random variation applied to the (non-backoff) delay, + # normalized to a (low, high) multiplier range (or None for no jitter). + # Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay + # randomization is on, or to no variation when it is off. + self._jitter: tuple[float, float] | None = self._normalize_jitter( + config.get( + "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 + ) + ) + self._delay_factor: float = float( + backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR")) + ) + self._max_delay: float = float( + backoff.get("max_delay", settings.getfloat("BACKOFF_MAX_DELAY")) + ) + self._min_delay: float = float( + backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY")) + ) + self._backoff_jitter: tuple[float, float] | None = self._normalize_jitter( + backoff.get("jitter", settings.getfloat("BACKOFF_JITTER")) + ) + # Which responses/exceptions trigger backoff is decided by the backoff + # middleware (see BackoffMiddleware), which reads the same per-scope + # "http_codes"/"exceptions" config and the global BACKOFF_* settings. + self._window: float = settings.getfloat("BACKOFF_WINDOW") + + # Concurrency. ``None`` means no scope-level limit (the downloader slots + # enforce concurrency instead); a limit is only set when configured + # explicitly. + configured_concurrency = config.get("concurrency") + if configured_concurrency is not None: + self._concurrency: int | None = int(configured_concurrency) + else: + self._concurrency = _default_scope_concurrency(settings) or None + # Used as the load denominator when the scope enforces no explicit + # concurrency limit (see get_load()). + self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") + + # Quota. + quota = config.get("quota") + self._quota: float | None = None if quota is None else float(quota) + self._quota_window: float = float( + config.get("window", settings.getfloat("THROTTLER_WINDOW")) + ) + + # State. + self._delay: float = self._base_delay + self._backoff_level: int = 0 + self._next_allowed_time: float | None = None + self._in_backoff_until: float | None = None + self._last_backoff_time: float | None = None + self._last_seen: float | None = None + self._active: int = 0 + self._slot_waiters: list[Deferred[None]] = [] + self._consumed: float = 0.0 + self._quota_window_start: float | None = None + + @staticmethod + def _now(now: float | None) -> float: + return time.monotonic() if now is None else now + + @staticmethod + def _normalize_jitter( + jitter: float | list[float], + ) -> tuple[float, float] | None: + """Normalize a ``jitter`` config value to a ``(low, high)`` multiplier + range, or ``None`` when no jitter applies. + + A scalar ``j`` means the symmetric range ``(-j, +j)``, so that + ``value * (1 + uniform(-j, +j))`` matches the historical + ``value * uniform(1 - j, 1 + j)``; a list/tuple is taken as an explicit + ``[low, high]`` range. + """ + if isinstance(jitter, (list, tuple)): + return (float(jitter[0]), float(jitter[1])) + if not jitter: + return None + return (-float(jitter), float(jitter)) + + @staticmethod + def _apply_jitter(value: float, jitter: tuple[float, float] | None) -> float: + if jitter is None: + return value + return value * (1 + random.uniform(*jitter)) # noqa: S311 + + def _effective_delay(self) -> float: + # ``self._delay`` is the deterministic delay (the base delay, or the + # bounded exponential value while backing off); jitter is applied here, + # per use, so successive delays spread out without compounding and + # without piling probability mass on the min/max bounds (which clipping + # a jittered value would do). While backing off, the backoff jitter + # applies; otherwise the plain delay jitter does. + if self._delay <= 0: + return self._delay + jitter = self._backoff_jitter if self._backoff_level > 0 else self._jitter + return self._apply_jitter(self._delay, jitter) + + def _recover(self, now: float) -> None: + if self._backoff_level == 0 or self._last_backoff_time is None: + return + if self._window <= 0: + # No window: no recovery cadence to step (would spin); recover at once. + self._backoff_level = 0 + self._delay = self._base_delay + self._in_backoff_until = None + self._last_backoff_time = None + return + while self._backoff_level > 0 and now - self._last_backoff_time >= self._window: + self._backoff_level -= 1 + self._last_backoff_time += self._window + if self._backoff_level == 0: + self._delay = self._base_delay + self._in_backoff_until = None + self._last_backoff_time = None + break + self._delay = max(self._base_delay, self._delay / self._delay_factor) + + def _maybe_reset_quota(self, now: float) -> None: + if self._quota is None: + return + if self._quota_window <= 0: + # No window: no reset cadence to step (would spin); keep it reset. + self._consumed = 0.0 + self._quota_window_start = now + return + if self._quota_window_start is None: + self._quota_window_start = now + return + while now - self._quota_window_start >= self._quota_window: + self._quota_window_start += self._quota_window + self._consumed = 0.0 + + def can_send(self, now: float | None = None, amount: float | None = None) -> float: + # can_send() only refreshes passive, time-based state (backoff recovery + # and the quota window) to reflect the current time. + now = self._now(now) + self._recover(now) + self._maybe_reset_quota(now) + waits = [0.0] + if self._in_backoff_until is not None: + waits.append(self._in_backoff_until - now) + if self._next_allowed_time is not None: + waits.append(self._next_allowed_time - now) + if self._quota is not None: + need = 0.0 if amount is None else float(amount) + # Block until the window resets only if some quota is already spent; + # a single oversized request is always allowed through. + if self._consumed > 0 and self._consumed + need > self._quota: + start = self._quota_window_start or now + waits.append(start + self._quota_window - now) + # Concurrency is enforced separately, via concurrency_blocked() and + # slot_event(), so acquire() can wait for a freed slot without polling. + return max(waits) + + def record_sent( + self, now: float | None = None, amount: float | None = None + ) -> None: + now = self._now(now) + self._last_seen = now + if self._in_backoff_until is not None and now >= self._in_backoff_until: + self._in_backoff_until = None + self._next_allowed_time = now + self._effective_delay() + self._active += 1 + if self._quota is not None and amount is not None: + self._maybe_reset_quota(now) + self._consumed += float(amount) + + def record_done(self, now: float | None = None) -> None: + if self._active > 0: + self._active -= 1 + self._fire_slot_waiters() + + def concurrency_blocked(self) -> bool: + return self._concurrency is not None and self._active >= self._concurrency + + def get_load(self) -> float: + limit = ( + self._concurrency + if self._concurrency is not None + else self._global_concurrency + ) + if not limit: + return 0.0 + return self._active / limit + + def slot_event(self) -> Deferred[None]: + """Return a Deferred that fires when a concurrency slot next frees up + (via :meth:`record_done`) or the limit is raised (via + :meth:`set_concurrency`).""" + event: Deferred[None] = Deferred() + self._slot_waiters.append(event) + return event + + def discard_slot_event(self, event: Deferred[None]) -> None: + with contextlib.suppress(ValueError): + self._slot_waiters.remove(event) + + def _fire_slot_waiters(self) -> None: + waiters, self._slot_waiters = self._slot_waiters, [] + for event in waiters: + if not event.called: + event.callback(None) + + def record_backoff( + self, + delay: float | None = None, + now: float | None = None, + cap: bool = True, + ) -> None: + now = self._now(now) + self._last_seen = now + self._last_backoff_time = now + self._backoff_level += 1 + if delay is not None: + # A hard delay (e.g. a Retry-After header) is a one-time gate: hold + # the scope back for at least this long *once*, matching the HTTP + # semantics of "do not retry before this time". It is deliberately + # not turned into the steady-state inter-request delay; the delay + # still grows by one exponential step below (and recovers over + # BACKOFF_WINDOW), so a small Retry-After does not become a long + # standing delay for every later request. + hard = min(float(delay), self._max_delay) if cap else float(delay) + self._in_backoff_until = now + hard + grown = self._delay * self._delay_factor if self._delay > 0 else self._min_delay + # Store the deterministic, bounded delay; jitter is applied per use in + # _effective_delay(), so it neither compounds across successive backoff + # steps nor concentrates on BACKOFF_MIN_DELAY / BACKOFF_MAX_DELAY. + self._delay = min(max(self._min_delay, grown), self._max_delay) + self._next_allowed_time = now + self._effective_delay() + + def reconcile_quota( + self, + consumed: float | None = None, + remaining: float | None = None, + now: float | None = None, + ) -> None: + if self._quota is None: + return + self._maybe_reset_quota(self._now(now)) + if remaining is not None: + self._consumed = max(0.0, self._quota - float(remaining)) + elif consumed is not None: + self._consumed = max(0.0, self._consumed + float(consumed)) + + def get_base_delay(self) -> float: + return self._base_delay + + def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None: + if only_increase and delay <= self._base_delay: + return + self._base_delay = delay + # Reflect the change in the effective delay unless a backoff is raising + # it above the base right now. + if self._backoff_level == 0: + self._delay = delay + + def set_concurrency(self, concurrency: int) -> None: + self._concurrency = max(1, int(concurrency)) + self._fire_slot_waiters() + + def is_idle(self, now: float, max_idle: float) -> bool: + if self._in_backoff_until is not None and self._in_backoff_until > now: + return False + if self._active > 0: + return False + if self._last_seen is None: + return True + return (now - self._last_seen) > max_idle diff --git a/scrapy/utils/_headers.py b/scrapy/utils/_headers.py new file mode 100644 index 000000000..21352858c --- /dev/null +++ b/scrapy/utils/_headers.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import datetime as dt +from email.utils import parsedate_to_datetime +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapy.http import Response + + +def _decoded_header(response: Response, name: str) -> str | None: + """Return the stripped UTF-8 value of the *name* header of *response*, or + ``None`` if it is absent or not valid UTF-8.""" + raw = response.headers.get(name) + if not raw: + return None + try: + return raw.decode("utf-8").strip() + except UnicodeDecodeError: + return None + + +def _parse_retry_after(response: Response) -> float | None: + value = _decoded_header(response, "Retry-After") + if value is None: + return None + if value.isdigit(): + return float(value) # seconds + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + return None + if date.tzinfo is None: + date = date.replace(tzinfo=dt.timezone.utc) + now = dt.datetime.now(dt.timezone.utc) + seconds_to_wait = (date - now).total_seconds() + # Keep sub-second precision (a date less than a second away must not be + # truncated to 0 and dropped); a past or present date yields no delay. + return max(0.0, seconds_to_wait) or None + + +def _parse_ratelimit_reset(response: Response) -> float | None: + value = _decoded_header(response, "RateLimit-Reset") + if value is None: + return None + try: + return float(value) + except ValueError: + return None diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..de16ee194 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -5,11 +5,11 @@ from __future__ import annotations import asyncio import logging import time -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Sequence from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -311,3 +311,106 @@ async def run_in_thread( from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 return await maybe_deferred_to_future(deferToThread(func, *args, **kwargs)) + + +async def sleep(seconds: float) -> None: + """Sleep for *seconds*, working in asyncio-reactor, non-asyncio-reactor, + and reactorless modes. + + Uses :func:`asyncio.sleep` when asyncio is available, and + :func:`~twisted.internet.task.deferLater` otherwise. + + .. versionadded:: VERSION + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + else: + from twisted.internet import reactor + + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + await maybe_deferred_to_future(deferLater(reactor, seconds, lambda: None)) + + +async def wait_for_first( + deferreds: Sequence[Deferred[Any]], + *, + timeout: float | None = None, +) -> tuple[set[Deferred[Any]], set[Deferred[Any]]]: + """Wait for the first of *deferreds* to fire, or until *timeout* seconds pass. + + Returns ``(done, pending)`` — two sets partitioning the input + :class:`~twisted.internet.defer.Deferred` objects — mirroring the API of + :func:`asyncio.wait`. + + Unfired deferreds in the ``pending`` set are neither cancelled nor + otherwise modified; the caller is responsible for any cleanup. + + A deferred that fails counts as done and its failure is **not** re-raised + here (it stays on the deferred for the caller to inspect or handle), exactly + as a failed awaitable lands in the ``done`` set of :func:`asyncio.wait`. + + Returns ``(set(), set())`` immediately when *deferreds* is empty. + + Works transparently in asyncio-reactor, non-asyncio-reactor, and + reactorless modes. + + .. versionadded:: VERSION + """ + if not deferreds: + return set(), set() + + if is_asyncio_available(): + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + future_to_deferred = {maybe_deferred_to_future(d): d for d in deferreds} + done_futures, pending_futures = await asyncio.wait( + list(future_to_deferred), + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + return ( + {future_to_deferred[f] for f in done_futures}, + {future_to_deferred[f] for f in pending_futures}, + ) + + from twisted.internet import reactor + + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + # Fire a single signal Deferred as soon as any input Deferred (or the + # timeout) completes, whether it succeeds or fails. Unlike a DeferredList + # with fireOnOneErrback (which would raise the first failure into this + # coroutine), _fire passes each result through untouched, so a failure just + # marks its Deferred as done and stays there for the caller — matching the + # asyncio branch and asyncio.wait(return_when=FIRST_COMPLETED). + signal: Deferred[None] = Deferred() + + def _fire(result: Any) -> Any: + if not signal.called: + signal.callback(None) + return result + + for d in deferreds: + d.addBoth(_fire) + + timeout_deferred: Deferred[Any] | None = None + if timeout is not None: + timeout_deferred = deferLater(reactor, timeout, lambda: None) + timeout_deferred.addBoth(_fire) + # Swallow the CancelledError from the finally-block cancel() below. + timeout_deferred.addErrback(lambda _: None) + + try: + await maybe_deferred_to_future(signal) + finally: + if timeout_deferred is not None and not timeout_deferred.called: + timeout_deferred.cancel() + + return ( + {d for d in deferreds if d.called}, + {d for d in deferreds if not d.called}, + ) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 90ed70262..4ac67da50 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -15,6 +15,7 @@ from twisted.web.client import Agent from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import default_settings from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed from scrapy.utils.spider import DefaultSpider @@ -66,13 +67,19 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - # When needed, useful settings can be added here, e.g. ones that prevent - # deprecation warnings. settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } + if prevent_warnings: + # Pin the per-scope concurrency to CONCURRENT_REQUESTS_PER_DOMAIN's + # default: keeps the suite's historical behavior and silences the + # warn-then-flip warning (see throttler._warn_on_deprecated_concurrency). + settings.setdefault( + "THROTTLER_SCOPE_CONCURRENCY", + default_settings.CONCURRENT_REQUESTS_PER_DOMAIN, + ) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..19272a0a9 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,7 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy.core.downloader import Downloader, Slot, tls +from scrapy.core.downloader import Downloader, Slot, _Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, _ScrapyClientContextFactory, @@ -41,8 +41,20 @@ if TYPE_CHECKING: class TestSlot: def test_repr(self): - slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)" + slot = _Slot() + assert repr(slot) == "_Slot()" + + def test_deprecated(self): + with pytest.warns(ScrapyDeprecationWarning, match="Slot class is deprecated"): + Slot() + + def test_deprecated_subclass(self): + with pytest.warns( + ScrapyDeprecationWarning, match="inherits from the deprecated Slot" + ): + + class MySlot(Slot): + pass @pytest.mark.requires_reactor # this test is related to the Twisted HTTP code diff --git a/tests/test_crawler.py b/tests/test_crawler.py index adac32df1..9b1264e76 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -5,6 +5,7 @@ import logging import re import signal import threading +import warnings from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar from unittest.mock import MagicMock @@ -52,6 +53,77 @@ def get_raw_crawler( return Crawler(spidercls or DefaultSpider, settings) +class TestScopeConcurrencyBridge: + """CONCURRENT_REQUESTS_PER_DOMAIN is deprecated and bridged into + THROTTLER_SCOPE_CONCURRENCY by setting priority (see + scrapy.throttler._default_scope_concurrency).""" + + @staticmethod + def _resolve(settings: Settings) -> int: + from scrapy.throttler import _default_scope_concurrency # noqa: PLC0415 + + return _default_scope_concurrency(settings) + + def test_neither_set_preserves_per_domain_default(self) -> None: + # Neither setting overridden: keep the historical per-domain default. + assert self._resolve(Settings()) == 8 + + def test_per_domain_overrides(self) -> None: + settings = Settings() + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") + assert self._resolve(settings) == 4 + + def test_scope_concurrency_overrides(self) -> None: + settings = Settings() + settings.set("THROTTLER_SCOPE_CONCURRENCY", 4, priority="project") + assert self._resolve(settings) == 4 + + def test_new_setting_wins_on_non_default_tie(self) -> None: + settings = Settings() + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") + settings.set("THROTTLER_SCOPE_CONCURRENCY", 9, priority="project") + assert self._resolve(settings) == 9 + + def test_higher_priority_deprecated_setting_wins(self) -> None: + settings = Settings() + settings.set("THROTTLER_SCOPE_CONCURRENCY", 9, priority="project") + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="cmdline") + assert self._resolve(settings) == 4 + + def test_warns_flip_when_neither_set(self) -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="Once CONCURRENT_REQUESTS_PER_DOMAIN is removed", + ): + get_crawler(prevent_warnings=False) + + def test_warns_deprecated_when_per_domain_set(self) -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated", + ): + get_crawler( + settings_dict={"CONCURRENT_REQUESTS_PER_DOMAIN": 4}, + prevent_warnings=False, + ) + + def test_no_concurrency_warning_when_scope_set(self) -> None: + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + get_crawler( + settings_dict={"THROTTLER_SCOPE_CONCURRENCY": 4}, + prevent_warnings=False, + ) + messages = [str(w.message) for w in recorded] + assert not any( + "Once CONCURRENT_REQUESTS_PER_DOMAIN is removed" in m for m in messages + ) + assert not any( + "CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated" in m + for m in messages + ) + + class TestBaseCrawler: @staticmethod def assertOptionIsDefault(settings: Settings, key: str) -> None: @@ -87,6 +159,30 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen + def test_spider_download_delay_deprecated(self) -> None: + class DelaySpider(DefaultSpider): + download_delay = 2.5 + + crawler = Crawler(DelaySpider) + with pytest.warns( + ScrapyDeprecationWarning, match="'download_delay' spider attribute" + ): + crawler._apply_settings() + assert crawler.settings.getfloat("DOWNLOAD_DELAY") == 2.5 + + def test_spider_download_delay_overridden_by_setting(self) -> None: + class DelaySpider(DefaultSpider): + download_delay = 2.5 + + crawler = Crawler(DelaySpider) + crawler.settings.set("DOWNLOAD_DELAY", 5.0, priority="spider") + with pytest.warns( + ScrapyDeprecationWarning, + match="'download_delay' spider attribute.*being ignored", + ): + crawler._apply_settings() + assert crawler.settings.getfloat("DOWNLOAD_DELAY") == 5.0 + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 082fc743e..061a551e5 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -74,6 +76,22 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + assert "crawler" not in calls[0] + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d58a6e09..80b204e4d 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,39 +1,32 @@ -import time from typing import Any +from urllib.parse import urlparse import pytest from scrapy import Request -from scrapy.core.downloader import Downloader, Slot +from scrapy.core.downloader import Downloader from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MetaSpider -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test class DownloaderSlotsSettingsTestSpider(MetaSpider): name = "downloader_slots" custom_settings = { - "DOWNLOAD_DELAY": 1, - "RANDOMIZE_DOWNLOAD_DELAY": False, "DOWNLOAD_SLOTS": { - "quotes.toscrape.com": { - "concurrency": 1, - "delay": 2, - "randomize_delay": False, - "throttle": False, - }, - "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + "quotes.toscrape.com": {"concurrency": 1}, + "books.toscrape.com": {"concurrency": 2}, }, } def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) assert self.mockserver - self.default_slot = self.mockserver.host + self.default_slot = urlparse(self.mockserver.url("/")).netloc self.times: dict[str, list[float]] = {} async def start(self): @@ -45,65 +38,84 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def parse(self, response): slot = response.meta.get("download_slot", self.default_slot) - self.times[slot].append(time.time()) + self.times[slot].append(response.meta.get("download_latency")) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): slot = response.meta.get("download_slot", self.default_slot) - self.times[slot].append(time.time()) - - -class TestCrawl: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_delay(self): - crawler = get_crawler(DownloaderSlotsSettingsTestSpider) - yield crawler.crawl(mockserver=self.mockserver) - slots = crawler.engine.downloader.slots - times = crawler.spider.times - tolerance = 0.3 - - delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = { - k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) - for k, v in slots.items() - } - - assert max(list(error_delta.values())) < tolerance + self.times[slot].append(response.meta.get("download_latency")) @coroutine_test -async def test_params(): - params = { - "concurrency": 1, - "delay": 2, - "randomize_delay": False, - } - settings = { - "DOWNLOAD_SLOTS": { - "example.com": params, - }, - } +async def test_concurrency_key_deprecated(): + settings = {"DOWNLOAD_SLOTS": {"example.com": {"concurrency": 3}}} crawler = get_crawler(DefaultSpider, settings_dict=settings) crawler.spider = crawler._create_spider() + with pytest.warns(ScrapyDeprecationWarning) as warns: + downloader = Downloader(crawler) + messages = [str(w.message) for w in warns] + assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) + downloader._get_slot(Request("https://example.com")) + downloader.close() + + +@coroutine_test +async def test_download_slots_deprecated(): + settings = {"DOWNLOAD_SLOTS": {"example.com": {"concurrency": 2}}} + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + with pytest.warns( + ScrapyDeprecationWarning, match="DOWNLOAD_SLOTS setting is deprecated" + ): + Downloader(crawler).close() + + +@coroutine_test +async def test_slots_deprecated(): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() downloader = Downloader(crawler) request = Request("https://example.com") - _, actual = downloader._get_slot(request) + request.meta[Downloader.DOWNLOAD_SLOT] = "example.com" + downloader.active.add(request) + with pytest.warns(ScrapyDeprecationWarning, match="Downloader.slots is deprecated"): + slot = downloader.slots.get("example.com") + assert slot is not None + assert isinstance(slot.active, set) + assert request in slot.active + downloader.active.discard(request) + downloader.close() + + +@coroutine_test +async def test_download_slot_meta_deprecated(): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + downloader = Downloader(crawler) + request = Request("https://example.com") + request.meta["download_slot"] = "custom" + with pytest.warns( + ScrapyDeprecationWarning, match="'download_slot' request meta key is deprecated" + ): + key, _ = downloader._get_slot(request) + downloader.close() + assert key == "custom" + + +@coroutine_test +async def test_delay_deprecated(): + settings = { + "DOWNLOAD_SLOTS": {"example.com": {"delay": 2, "randomize_delay": False}} + } + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + with pytest.warns(ScrapyDeprecationWarning) as warns: + downloader = Downloader(crawler) + messages = [str(w.message) for w in warns] + assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) + downloader._get_slot(Request("https://example.com")) downloader.close() - expected = Slot(**params) - for param in params: - assert getattr(expected, param) == getattr(actual, param), ( - f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" - ) @coroutine_test @@ -122,7 +134,7 @@ async def test_get_slot_deprecated_spider_arg(): downloader.close() assert key1 == key2 - assert slot1 == slot2 + assert slot1._key == slot2._key @pytest.mark.parametrize( @@ -132,6 +144,7 @@ async def test_get_slot_deprecated_spider_arg(): "scrapy.pqueues.DownloaderAwarePriorityQueue", ], ) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @coroutine_test async def test_none_slot_with_priority_queue( mockserver: MockServer, priority_queue_class: str diff --git a/tests/test_engine.py b/tests/test_engine.py index e51eb4664..e4f44ca0a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -595,6 +595,21 @@ class TestEngineDownloadAsync: engine._slot.add_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request) + @coroutine_test + async def test_download_async_fetch_needs_spider(self, engine): + """A downloader whose fetch() requires a spider gets it passed in.""" + engine._downloader_fetch_needs_spider = True + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, request) + assert result == response + engine.downloader.fetch.assert_called_once_with(request, engine.spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestEngineDownload(TestEngineDownloadAsync): @@ -640,6 +655,154 @@ async def test_request_scheduled_signal(): crawler.signals.disconnect(signal_handler, signals.request_scheduled) +class TestEngineThrottler: + @pytest.fixture + def engine(self): + crawler = get_crawler(MySpider) + engine = ExecutionEngine(crawler, lambda _: None) + yield engine + engine.downloader.close() + + def test_pause_cancels_delay_wakeup(self, engine): + wakeup = Mock() + engine._delay_wakeup = wakeup + engine.pause() + assert engine.paused is True + wakeup.cancel.assert_called_once_with() + assert engine._delay_wakeup is None + engine.unpause() + assert engine.paused is False + + @pytest.mark.requires_reactor # call_later() needs a reactor or asyncio loop + def test_maybe_arm_delay_wakeup_arms_timer(self, engine): + engine._get_next_request_delay = lambda: 5.0 + engine._slot = Mock() + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is not None + # Cancel the scheduled reactor call so it does not leak into other tests. + engine._cancel_delay_wakeup() + + def test_maybe_arm_delay_wakeup_no_delay(self, engine): + engine._get_next_request_delay = lambda: None + engine._slot = Mock() + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is None + + def test_maybe_arm_delay_wakeup_zero_delay(self, engine): + # A 0 delay means a request is ready but could not be sent (e.g. the + # downloader is at capacity); arming a 0-second timer would busy-loop + # the engine, so no timer must be armed. + engine._get_next_request_delay = lambda: 0.0 + engine._slot = Mock() + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is None + + def test_maybe_arm_delay_wakeup_not_supported(self, engine): + # A scheduler without get_next_request_delay never arms a timer. + engine._get_next_request_delay = None + engine._slot = Mock() + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is None + + def test_maybe_warn_throttler_backout(self, engine): + # A scheduler without get_next_request_delay is not throttler-aware, so + # the warning recommends switching to one. + engine._get_next_request_delay = None + with LogCapture() as log: + engine._maybe_warn_throttler_backout() + # A second call is a no-op (the warning is emitted only once). + engine._maybe_warn_throttler_backout() + assert engine._throttler_backout_warned is True + assert str(log).count("ThrottlerAwareScheduler") == 1 + + def test_maybe_warn_throttler_backout_throttler_aware(self, engine): + # A throttler-aware scheduler (one with get_next_request_delay) holds + # throttled requests itself, so no warning is emitted. + engine._get_next_request_delay = lambda: None + with LogCapture() as log: + engine._maybe_warn_throttler_backout() + assert engine._throttler_backout_warned is False + assert "ThrottlerAwareScheduler" not in str(log) + + def test_spider_is_idle_false_while_scheduling(self, engine): + engine._slot = Mock() + engine.scraper.slot = Mock() + engine.scraper.slot.is_idle.return_value = True + engine.downloader = Mock() + engine.downloader.active = [] + engine._throttler_waiting = set() + engine._start = None + engine._scheduling = 1 + # An in-flight async enqueue keeps the spider from being considered idle. + assert engine.spider_is_idle() is False + + @coroutine_test + async def test_enqueue_request_async_dropped(self, engine): + scheduler = Mock() + + async def enqueue_request_async(request): + return False + + scheduler.enqueue_request_async = enqueue_request_async + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine.spider = Mock() + dropped = [] + + def on_dropped(request, spider): + dropped.append(request) + + engine.signals.connect(on_dropped, signals.request_dropped, weak=False) + engine._scheduling = 1 + request = Request("http://a.example") + await engine._enqueue_request_async(request) + assert dropped == [request] + assert engine._scheduling == 0 + engine._slot.nextcall.schedule.assert_called_once_with() + + @coroutine_test + async def test_enqueue_request_async_error(self, engine): + scheduler = Mock() + + async def enqueue_request_async(request): + raise RuntimeError("boom") + + scheduler.enqueue_request_async = enqueue_request_async + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine.spider = Mock() + engine._scheduling = 1 + with LogCapture() as log: + await engine._enqueue_request_async(Request("http://a.example")) + assert "Error while enqueuing request" in str(log) + assert engine._scheduling == 0 + engine._slot.nextcall.schedule.assert_called_once_with() + + @coroutine_test + async def test_enqueue_request_async_slot_gone(self, engine): + scheduler = Mock() + + async def enqueue_request_async(request): + # The spider is closed while the enqueue is in flight. + engine._slot = None + return True + + scheduler.enqueue_request_async = enqueue_request_async + slot = Mock() + slot.scheduler = scheduler + engine._slot = slot + engine.spider = Mock() + engine._scheduling = 1 + await engine._enqueue_request_async(Request("http://a.example")) + assert engine._scheduling == 0 + # No reschedule is attempted once the slot is gone. + slot.nextcall.schedule.assert_not_called() + + class TestEngineCloseSpider: """Tests for exception handling coverage during close_spider_async().""" @@ -688,7 +851,7 @@ class TestEngineCloseSpider: engine = ExecutionEngine(crawler, lambda _: None) crawler.engine = engine await engine.open_spider_async() - del engine.downloader.slots + engine.downloader.close = Mock(side_effect=Exception("close failed")) await engine.close_spider_async() assert "Downloader close failure" in caplog.text diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 6c6a6584a..cff66bb1e 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -4,14 +4,19 @@ from unittest.mock import Mock import pytest import queuelib -from scrapy.core.downloader import Downloader from scrapy.http.request import Request -from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue +from scrapy.pqueues import ( + DownloaderAwarePriorityQueue, + ScrapyPriorityQueue, + ThrottlerAwarePriorityQueue, +) from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue +from scrapy.throttler import iter_scopes from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader +from tests.utils.decorators import coroutine_test class TestPriorityQueue: @@ -186,22 +191,22 @@ class TestDownloaderAwarePriorityQueue: # No active downloads are tracked in the downloader, so every slot has # the same score and tie-breaking must not starve a slot. req_a1 = Request("https://example.org/a1") - req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a1.meta["throttler_scopes"] = "slot-a" req_b1 = Request("https://example.org/b1") - req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b1.meta["throttler_scopes"] = "slot-b" req_a2 = Request("https://example.org/a2") - req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a2.meta["throttler_scopes"] = "slot-a" req_b2 = Request("https://example.org/b2") - req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b2.meta["throttler_scopes"] = "slot-b" for request in (req_a1, req_b1, req_a2, req_b2): self.queue.push(request) slots = [ - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], ] assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"] @@ -210,48 +215,49 @@ class TestDownloaderAwarePriorityQueue: # If the selected slot becomes empty, rotation should continue from # that slot marker to avoid restarting from the smallest slot. req_a1 = Request("https://example.org/a1") - req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a1.meta["throttler_scopes"] = "slot-a" req_a2 = Request("https://example.org/a2") - req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a2.meta["throttler_scopes"] = "slot-a" req_b1 = Request("https://example.org/b1") - req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b1.meta["throttler_scopes"] = "slot-b" req_c1 = Request("https://example.org/c1") - req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + req_c1.meta["throttler_scopes"] = "slot-c" for request in (req_a1, req_a2, req_b1, req_c1): self.queue.push(request) slots = [ - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], ] assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"] def test_pop_prefers_slot_with_fewer_active_downloads(self): - downloader = self.queue._downloader_interface.downloader + throttler = self.queue._downloader_interface._throttler + assert throttler is not None req_a = Request("https://example.org/a") - req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a.meta["throttler_scopes"] = "slot-a" req_b = Request("https://example.org/b") - req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b.meta["throttler_scopes"] = "slot-b" req_c = Request("https://example.org/c") - req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + req_c.meta["throttler_scopes"] = "slot-c" for req in (req_a, req_b, req_c): self.queue.push(req) - downloader.increment("slot-a") - downloader.increment("slot-c") + throttler._get_scope_manager("slot-a")._active = 1 + throttler._get_scope_manager("slot-c")._active = 1 popped = self.queue.pop() assert popped.url == req_b.url def test_contains(self): req = Request("https://example.org/") - req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot" + req.meta["throttler_scopes"] = "example-slot" assert "example-slot" not in self.queue self.queue.push(req) assert "example-slot" in self.queue @@ -309,3 +315,263 @@ def test_pop_order(input_, output): actual_output_urls.append(request.url) assert actual_output_urls == expected_output_urls + + +class TestThrottlerAwarePriorityQueue: + def _queue(self, crawler, key=""): + return build_from_crawler( + ThrottlerAwarePriorityQueue, + crawler, + downstream_queue_cls=FifoMemoryQueue, + key=key, + ) + + async def _push(self, queue, crawler, request): + scope_set = frozenset(iter_scopes(await crawler.throttler.get_scopes(request))) + queue.push(request, scope_set) + + @coroutine_test + async def test_partitions_by_scope_set(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://a.com/1")) + await self._push(queue, crawler, Request("http://a.com/2")) + await self._push(queue, crawler, Request("http://b.com/1")) + # Two distinct scope sets -> two internal queues, three requests. + assert len(queue.pqueues) == 2 + assert len(queue) == 3 + + @coroutine_test + async def test_pop_skips_blocked_scope(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLER_SCOPES": {"slow.com": {"delay": 1000.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + }, + ) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://slow.com/1")) + await self._push(queue, crawler, Request("http://slow.com/2")) + await self._push(queue, crawler, Request("http://fast.com/1")) + # The first slow request is sendable (no delay accrued yet); after it is + # popped (and reserved), the second slow request is blocked, but the + # fast one is still served. + popped = [queue.pop(), queue.pop(), queue.pop()] + urls = [r.url if r else None for r in popped] + assert "http://fast.com/1" in urls + assert "http://slow.com/1" in urls + # The blocked second slow request stays in the queue. + assert None in urls + assert len(queue) == 1 + delay = queue.get_next_request_delay() + assert delay is not None + assert delay == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_pop_holds_request_with_delay(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + await self._push( + queue, + crawler, + Request("http://slow.com/1", meta={"delay": 1000.0}), + ) + await self._push(queue, crawler, Request("http://fast.com/1")) + # The delayed request is held back even though its scope is otherwise + # unconstrained; the request without a delay is served. + popped = [queue.pop(), queue.pop()] + urls = [r.url if r else None for r in popped] + assert "http://fast.com/1" in urls + assert None in urls + assert len(queue) == 1 + delay = queue.get_next_request_delay() + assert delay is not None + assert delay == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_delayed_request_does_not_block_scope_set(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + # Both requests share the same (example.com) scope set; only the first + # carries a per-request delay. + await self._push( + queue, + crawler, + Request("http://example.com/slow", meta={"delay": 1000.0}), + ) + await self._push(queue, crawler, Request("http://example.com/fast")) + # The delayed request is held aside, so the other request in the same + # scope set is served right away instead of being stuck behind it. + assert queue.pop().url == "http://example.com/fast" + # The delayed request is not lost, just not poppable yet. + assert queue.pop() is None + assert len(queue) == 1 + assert queue.get_next_request_delay() == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_delayed_request_promoted_when_due(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + request = Request("http://example.com/slow", meta={"delay": 1000.0}) + await self._push(queue, crawler, request) + assert queue.pop() is None # held back by its per-request delay + # Once the delay elapses the request is promoted into its scope-set + # queue, served, and flagged so the delay is not applied a second time. + queue._promote_ready(queue._delayed[0][0]) + popped = queue.pop() + assert popped is request + assert popped.meta["_throttler_delayed"] is True + assert len(queue) == 0 + + @coroutine_test + async def test_delayed_request_persisted_on_close(self): + # With a JOBDIR (disk queue), a request held back by its per-request + # delay must not be lost on a graceful stop: close() flushes it to disk + # so it is restored on resume. + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + temp_dir = tempfile.mkdtemp() + queue = build_from_crawler( + ThrottlerAwarePriorityQueue, + crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=temp_dir, + ) + await self._push( + queue, + crawler, + Request("http://example.com/slow", meta={"delay": 1000.0}), + ) + assert len(queue) == 1 # held in memory, not yet in any scope-set queue + state = queue.close() # graceful stop + + resumed = build_from_crawler( + ThrottlerAwarePriorityQueue, + crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=temp_dir, + startprios=state, + ) + assert len(resumed) == 1 + popped = resumed.pop() + assert popped is not None + assert popped.url == "http://example.com/slow" + # Its delay is marked consumed, so it does not re-block on resume. + assert popped.meta["_throttler_delayed"] is True + resumed.close() + + @coroutine_test + async def test_least_loaded_first(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLER_SCOPES": { + "a.com": {"concurrency": 4}, + "b.com": {"concurrency": 4}, + } + }, + ) + queue = self._queue(crawler) + # Make a.com busier than b.com. + busy = Request("http://a.com/0") + await crawler.throttler.get_scopes(busy) + crawler.throttler.reserve(busy) + await self._push(queue, crawler, Request("http://a.com/1")) + await self._push(queue, crawler, Request("http://b.com/1")) + # Equal priority, so the lower-load scope (b.com) is served first. + assert queue.pop().url == "http://b.com/1" + + @coroutine_test + async def test_priority_beats_load(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLER_SCOPES": { + "a.com": {"concurrency": 4}, + "b.com": {"concurrency": 4}, + } + }, + ) + queue = self._queue(crawler) + # Make a.com busier than b.com. + busy = Request("http://a.com/0") + await crawler.throttler.get_scopes(busy) + crawler.throttler.reserve(busy) + # The a.com request has higher priority, and a.com still has room, so it + # is served first despite a.com being the busier scope. + await self._push(queue, crawler, Request("http://a.com/1", priority=10)) + await self._push(queue, crawler, Request("http://b.com/1")) + assert queue.pop().url == "http://a.com/1" + + @coroutine_test + async def test_empty_and_close(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + assert queue.pop() is None + assert queue.get_next_request_delay() is None + await self._push(queue, crawler, Request("http://a.com/1")) + assert queue.close() != {} + + @coroutine_test + async def test_get_next_request_delay_zero_when_ready(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://a.com/1")) + # A sendable head means no wait is needed. + assert queue.get_next_request_delay() == 0.0 + + @coroutine_test + async def test_get_next_request_delay_ignores_empty_queues(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + # An empty (but still registered) internal queue is skipped. + queue.pqueues[frozenset({"a.com"})] = queue._pqfactory(frozenset({"a.com"})) + assert queue.get_next_request_delay() is None + + @coroutine_test + async def test_get_next_request_delay_keeps_minimum(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLER_SCOPES": { + "a.com": {"delay": 10.0}, + "b.com": {"delay": 1000.0}, + }, + "RANDOMIZE_DOWNLOAD_DELAY": False, + }, + ) + queue = self._queue(crawler) + # Two requests per scope so a blocked head remains after the first one + # (sendable, since no delay has accrued yet) is popped and reserved. + await self._push(queue, crawler, Request("http://a.com/1")) + await self._push(queue, crawler, Request("http://a.com/2")) + await self._push(queue, crawler, Request("http://b.com/1")) + await self._push(queue, crawler, Request("http://b.com/2")) + queue.pop() + queue.pop() + # Both scopes are now time-blocked; the smaller per-scope delay wins, + # so the larger one exercises the "not below the running minimum" branch. + delay = queue.get_next_request_delay() + assert delay == pytest.approx(10.0, abs=1.0) + + @coroutine_test + async def test_pop_handles_drained_selected_queue(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://a.com/1")) + inner = next(iter(queue.pqueues.values())) + # _select() still reports a sendable head, but pop() yields nothing: the + # request-is-None guard must not try to reserve a missing request. + inner.pop = lambda: None + assert queue.pop() is None + + def test_non_dict_slot_startprios(self): + crawler = get_crawler(Spider) + with pytest.raises(ValueError, match="slot_startprios"): + build_from_crawler( + ThrottlerAwarePriorityQueue, + crawler, + downstream_queue_cls=FifoMemoryQueue, + key="", + startprios=[1, 2, 3], + ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 5249736f2..1062c2d5c 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -88,6 +88,16 @@ class BaseRobotParserTest: assert rp.allowed("https://site.local/index.html", "*") assert rp.allowed("https://site.local/disallowed", "*") + def test_crawl_delay(self): + robotstxt_body = b"User-agent: *\nCrawl-delay: 10\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") == 10.0 + + def test_crawl_delay_unset(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") is None + def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ User-Agent: * @@ -171,6 +181,9 @@ class TestRerpRobotParser(BaseRobotParserTest): def test_length_based_precedence(self): pytest.skip("Rerp does not support length based directives precedence.") + def test_crawl_delay(self): + pytest.skip("Rerp does not expose Crawl-delay directives.") + class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8637de41e..05dda6026 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import warnings from abc import ABC, abstractmethod from collections import deque @@ -10,7 +11,7 @@ from unittest.mock import Mock import pytest from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import BaseScheduler, Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler, ThrottlerAwareScheduler from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -26,6 +27,8 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path + from scrapy.http.request import CallbackT + class MemoryScheduler(BaseScheduler): paused = False @@ -406,3 +409,251 @@ class TestIncompatibility: ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" ): self._incompatible() + + +_THROTTLER_AWARE_PQ = "scrapy.pqueues.ThrottlerAwarePriorityQueue" + + +class TestThrottlerAwareScheduler: + def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler: + settings = { + "SCHEDULER_PRIORITY_QUEUE": _THROTTLER_AWARE_PQ, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + **(settings_dict or {}), + } + return get_crawler(Spider, settings) + + def _scheduler(self, crawler: Crawler) -> ThrottlerAwareScheduler: + spider = Spider(name="spider") + crawler.spider = spider + scheduler = ThrottlerAwareScheduler.from_crawler(crawler) + scheduler.open(spider) + return scheduler + + @coroutine_test + async def test_enqueue_async_and_dequeue(self) -> None: + scheduler = self._scheduler(self._crawler()) + assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True + assert scheduler.has_pending_requests() + assert len(scheduler) == 1 + request = scheduler.next_request() + assert request is not None + assert request.url == "http://a.com/1" + assert scheduler.next_request() is None + assert not scheduler.has_pending_requests() + scheduler.close("finished") + + def test_sync_enqueue_raises(self) -> None: + scheduler = self._scheduler(self._crawler()) + with pytest.raises(RuntimeError, match="asynchronous enqueue path"): + scheduler.enqueue_request(Request("http://a.com/1")) + scheduler.close("finished") + + def test_requires_throttler_aware_priority_queue(self) -> None: + crawler = self._crawler( + {"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ScrapyPriorityQueue"} + ) + spider = Spider(name="spider") + crawler.spider = spider + scheduler = ThrottlerAwareScheduler.from_crawler(crawler) + with pytest.raises(ValueError, match="throttler-aware priority queue"): + scheduler.open(spider) + + @coroutine_test + async def test_delay_blocks_and_reports_delay(self) -> None: + crawler = self._crawler( + { + "THROTTLER_SCOPES": {"slow.com": {"delay": 1000.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + }, + ) + scheduler = self._scheduler(crawler) + await scheduler.enqueue_request_async(Request("http://slow.com/1")) + await scheduler.enqueue_request_async(Request("http://slow.com/2")) + # The first request is sendable; the second is blocked by the delay. + first = scheduler.next_request() + assert first is not None + assert scheduler.next_request() is None + assert scheduler.has_pending_requests() + assert scheduler.get_next_request_delay() == pytest.approx(1000.0, abs=1.0) + scheduler.close("finished") + + @coroutine_test + async def test_no_delay_when_only_concurrency_blocked(self) -> None: + crawler = self._crawler( + {"THROTTLER_SCOPES": {"slow.com": {"concurrency": 1}}}, + ) + scheduler = self._scheduler(crawler) + await scheduler.enqueue_request_async(Request("http://slow.com/1")) + await scheduler.enqueue_request_async(Request("http://slow.com/2")) + assert scheduler.next_request() is not None + assert scheduler.next_request() is None + # A purely concurrency-blocked state has no time-based wakeup. + assert scheduler.get_next_request_delay() is None + scheduler.close("finished") + + @coroutine_test + async def test_delayed_request_survives_jobdir_stop(self, tmp_path: Path) -> None: + # A request held back by its per-request delay must not be + # lost on a graceful stop when a JOBDIR is configured: it is flushed to + # the disk queue on close and restored on resume. + crawler = self._crawler( + {"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + scheduler = self._scheduler(crawler) + request = Request("http://a.com/slow", meta={"delay": 1000.0}) + assert await scheduler.enqueue_request_async(request) is True + assert len(scheduler) == 1 + # The delay holds it back, so nothing is dequeued before the stop. + assert scheduler.next_request() is None + scheduler.close("finished") + + # Resume from the same JOBDIR: the request is still there and, having + # been held once, is now sendable. + resumed = self._scheduler( + self._crawler({"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False}) + ) + assert len(resumed) == 1 + resumed_request = resumed.next_request() + assert resumed_request is not None + assert resumed_request.url == "http://a.com/slow" + resumed.close("finished") + + @coroutine_test + async def test_enqueue_async_filters_duplicates(self) -> None: + crawler = self._crawler( + {"DUPEFILTER_CLASS": "scrapy.dupefilters.RFPDupeFilter"} + ) + scheduler = self._scheduler(crawler) + assert crawler.spider is not None + crawler.spider.crawler = crawler # the dupefilter logs via spider.crawler + assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True + # The same request is filtered out the second time around. + assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is False + assert len(scheduler) == 1 + scheduler.close("finished") + + @coroutine_test + async def test_enqueue_async_unserializable_falls_back_to_memory( + self, tmp_path: Path, caplog + ) -> None: + crawler = self._crawler({"JOBDIR": str(tmp_path), "SCHEDULER_DEBUG": True}) + scheduler = self._scheduler(crawler) + # A lambda callback cannot be serialized to disk, so the request falls + # back to the in-memory queue and the failure is logged once. + request = Request( + "http://a.com/1", callback=cast("CallbackT", lambda response: None) + ) + with caplog.at_level(logging.WARNING, logger="scrapy.core.scheduler"): + assert await scheduler.enqueue_request_async(request) is True + assert "Unable to serialize request" in caplog.text + assert crawler.stats is not None + assert crawler.stats.get_value("scheduler/unserializable") == 1 + assert crawler.stats.get_value("scheduler/enqueued/memory") == 1 + scheduler.close("finished") + + @coroutine_test + async def test_enqueue_async_unserializable_without_debug( + self, tmp_path: Path + ) -> None: + # Same fallback as above, but with SCHEDULER_DEBUG off the failure is + # tracked in stats without logging a warning. + crawler = self._crawler({"JOBDIR": str(tmp_path)}) + scheduler = self._scheduler(crawler) + request = Request( + "http://a.com/1", callback=cast("CallbackT", lambda response: None) + ) + assert await scheduler.enqueue_request_async(request) is True + assert crawler.stats is not None + assert crawler.stats.get_value("scheduler/unserializable") == 1 + scheduler.close("finished") + + @coroutine_test + async def test_resume_from_disk(self, tmp_path: Path) -> None: + settings = {"JOBDIR": str(tmp_path)} + scheduler = self._scheduler(self._crawler(settings)) + await scheduler.enqueue_request_async(Request("http://a.com/1")) + await scheduler.enqueue_request_async(Request("http://b.com/1")) + scheduler.close("shutdown") + + scheduler2 = self._scheduler(self._crawler(settings)) + assert len(scheduler2) == 2 + urls = set() + while (request := scheduler2.next_request()) is not None: + urls.add(request.url) + assert urls == {"http://a.com/1", "http://b.com/1"} + scheduler2.close("finished") + + +class TestIntegrationWithThrottlerAwareScheduler: + @inline_callbacks_test + def test_integration(self): + crawler = get_crawler( + spidercls=StartUrlsSpider, + settings_dict={ + "SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue", + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + }, + ) + with MockServer() as mockserver: + url = mockserver.url("/status?n=200", is_secure=False) + start_urls = [url] * 6 + yield crawler.crawl(start_urls) + assert crawler.stats.get_value("downloader/response_count") == len( + start_urls + ) + + @inline_callbacks_test + def test_integration_follow_requests(self): + # Exercises the engine's asynchronous enqueue path for requests yielded + # from a callback (not just start requests), and the idle guard that + # keeps the spider open while an enqueue is in flight. + class FollowSpider(Spider): + name = "follow" + + def __init__(self, base_url, **kwargs): + self.base_url = base_url + super().__init__(**kwargs) + + async def start(self): + yield Request(self.base_url + "/status?n=200") + + def parse(self, response): + if b"follow" not in response.url.encode(): + yield Request(response.url + "&follow=1", callback=self.parse) + + with MockServer() as mockserver: + base_url = mockserver.url("", is_secure=False).rstrip("/") + crawler = get_crawler( + spidercls=FollowSpider, + settings_dict={ + "SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue", + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + }, + ) + yield crawler.crawl(base_url=base_url) + assert crawler.stats.get_value("downloader/response_count") == 2 + + @inline_callbacks_test + def test_integration_with_delay(self): + # A small per-scope delay forces the engine to arm the throttler wakeup + # timer between requests; the crawl must still complete. + crawler = get_crawler( + spidercls=StartUrlsSpider, + settings_dict={ + "SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue", + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "RANDOMIZE_DOWNLOAD_DELAY": False, + "THROTTLER_SCOPES": {"127.0.0.1": {"delay": 0.05}}, + }, + ) + with MockServer() as mockserver: + url = mockserver.url("/status?n=200", is_secure=False) + start_urls = [url] * 4 + yield crawler.crawl(start_urls) + assert crawler.stats.get_value("downloader/response_count") == len( + start_urls + ) diff --git a/tests/test_throttler.py b/tests/test_throttler.py new file mode 100644 index 000000000..503ecc26d --- /dev/null +++ b/tests/test_throttler.py @@ -0,0 +1,1173 @@ +from __future__ import annotations + +import logging + +import pytest + +from scrapy import signals +from scrapy.exceptions import DownloadTimeoutError +from scrapy.http import Request, Response +from scrapy.settings import default_settings +from scrapy.throttler import ( + Throttler, + ThrottlerScopeManager, + _add_bare_scope, + _parse_ratelimit_reset, + _parse_retry_after, + _to_scope_dict, + add_scope, + iter_scope_values, + iter_scopes, + scope_cache, + update_scope_backoff, +) +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider +from tests.utils.decorators import coroutine_test + + +def _manager(settings=None): + crawler = get_crawler(settings_dict=settings) + return Throttler.from_crawler(crawler) + + +def _scope_manager(settings=None, config=None): + crawler = get_crawler(settings_dict=settings) + return ThrottlerScopeManager.from_crawler(crawler, config or {"id": "example.com"}) + + +class _FakeRobotParser: + """A minimal robots.txt parser stub for :signal:`robots_parsed` tests. + + *delay* is returned by :meth:`crawl_delay`, unless it is an exception, in + which case it is raised to emulate a backend-specific failure. + """ + + def __init__(self, delay): + self._delay = delay + + def crawl_delay(self, useragent): + if isinstance(self._delay, Exception): + raise self._delay + return self._delay + + +def _response(status=200, headers=None, url="http://example.com", meta=None): + request = Request(url, meta=meta or {}) + return Response(url, status=status, headers=headers or {}, request=request) + + +def test_deprecated_concurrency_defaults_differ(): + """``_warn_on_deprecated_concurrency`` emits a warn-then-flip message that + only makes sense while the two concurrency defaults differ (otherwise it + reads "will drop from N to N"). Guard that invariant here so that lowering + the deprecated default to match is caught by the test suite instead of + shipping a bogus warning or aborting a crawl.""" + assert ( + default_settings.CONCURRENT_REQUESTS_PER_DOMAIN + != default_settings.THROTTLER_SCOPE_CONCURRENCY + ) + + +class TestThrottler: + @coroutine_test + async def test_get_scopes_returns_netloc(self): + manager = _manager() + assert ( + await manager.get_scopes(Request("http://example.com/a")) == "example.com" + ) + + @coroutine_test + async def test_get_scopes_cached(self): + manager = _manager() + request = Request("http://example.com/a") + first = await manager.get_scopes(request) + # get_scopes is deterministic per request, so a second call yields the + # same scopes. + assert await manager.get_scopes(request) == first + + @coroutine_test + async def test_get_scopes_meta_string(self): + manager = _manager() + request = Request("http://example.com/a", meta={"throttler_scopes": "api"}) + assert await manager.get_scopes(request) == "api" + + @coroutine_test + async def test_get_scopes_meta_dict(self): + manager = _manager() + request = Request( + "http://example.com/a", meta={"throttler_scopes": {"api": 2.0}} + ) + assert await manager.get_scopes(request) == {"api": 2.0} + + @coroutine_test + async def test_get_scopes_persisted_in_meta(self): + from scrapy.throttler import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 + + manager = _manager() + request = Request("http://example.com/a") + scopes = await manager.get_scopes(request) + assert request.meta[_RESOLVED_SCOPES_META_KEY] == scopes + + @coroutine_test + async def test_scope_cache_works_without_crawler(self): + # scope_cache only persists to meta; it needs nothing from the manager. + from scrapy.throttler import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 + + class CrawlerlessManager: + @scope_cache + async def get_scopes(self, request): + return "scope" + + request = Request("http://example.com/a") + assert await CrawlerlessManager().get_scopes(request) == "scope" + assert request.meta[_RESOLVED_SCOPES_META_KEY] == "scope" + + @coroutine_test + async def test_backoff_reuses_persisted_scopes(self): + # Once get_scopes has resolved and persisted the scopes, the backoff + # path reuses them instead of resolving again. + calls = [] + + class CountingManager(Throttler): + @scope_cache + async def get_scopes(self, request): + calls.append(request.url) + return urlparse_cached(request).netloc + + manager = CountingManager( + get_crawler(settings_dict={"BACKOFF_EXCEPTIONS": ["builtins.ValueError"]}) + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + assert calls == ["http://example.com/a"] + assert ( + await manager.get_exception_backoff(request, ValueError()) == "example.com" + ) + # No second resolution. + assert calls == ["http://example.com/a"] + + @coroutine_test + async def test_get_scopes_survives_disk_roundtrip(self): + from scrapy.utils.request import request_from_dict # noqa: PLC0415 + + manager = _manager() + request = Request( + "http://example.com/a", meta={"throttler_scopes": {"bucket": 3.0}} + ) + await manager.get_scopes(request) + # A request restored from a disk queue is a fresh object; the synchronous + # readiness path must still recover its scopes (with quota values) from + # the persisted meta, without re-running get_scopes. + restored = request_from_dict(request.to_dict()) + assert manager._cached_scope_values(restored) == [("bucket", 3.0)] + + @coroutine_test + async def test_get_scopes_reresolved_after_cross_host_replace(self): + # A redirect built with Request.replace() copies meta (including the + # persisted scopes), but get_scopes always re-resolves (it never reads + # the persisted value back), so it must not reuse the original host's + # scopes. + manager = _manager() + request = Request("http://example.com/a") + assert await manager.get_scopes(request) == "example.com" + redirected = request.replace(url="http://other.example/a") + assert await manager.get_scopes(redirected) == "other.example" + + @coroutine_test + async def test_get_initial_backoff_none(self): + manager = _manager() + assert await manager.get_initial_backoff() is None + + def test_scope_manager_class_in_config(self): + manager = _manager( + {"THROTTLER_SCOPES": {"example.com": {"manager": ThrottlerScopeManager}}} + ) + scope = manager._get_scope_manager("example.com") + assert isinstance(scope, ThrottlerScopeManager) + + def test_release_frees_concurrency(self): + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}}) + scope = manager._get_scope_manager("example.com") + request = Request("http://example.com") + scope.record_sent(now=0.0) + manager._reserved[request] = [(scope, None)] + assert scope.concurrency_blocked() is True + manager.release(request) + assert scope.concurrency_blocked() is False + # Releasing again is a no-op. + manager.release(request) + assert scope.concurrency_blocked() is False + + @coroutine_test + async def test_acquire_waits_for_freed_slot(self): + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}}) + r1 = Request("http://example.com/1") + r2 = Request("http://example.com/2") + # Drive acquire() the way the engine does, so it runs as a real task that + # can await the slot event under the asyncio reactor. + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r1))) + scope = manager._get_scope_manager("example.com") + assert scope.concurrency_blocked() is True + assert scope.concurrency_blocked() is True + # acquire(r2) must block until r1 frees the slot; release it on the next + # event loop tick so the event-driven wait wakes up. + call_later(0, manager.release, r1) + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r2))) + assert scope.concurrency_blocked() is True + assert r2 in manager._reserved + + def test_apply_backoff_reconciles_quota_without_backoff(self): + manager = _manager({"THROTTLER_SCOPES": {"cost": {"quota": 100.0}}}) + scope = manager._get_scope_manager("cost") + manager._apply_backoff({"cost": {"consumed": 5.0}}) + # Quota was reconciled but no backoff step was applied. + assert scope._consumed == pytest.approx(5.0) + assert scope._backoff_level == 0 + + def test_apply_backoff_delay_and_consumed(self): + manager = _manager({"THROTTLER_SCOPES": {"cost": {"quota": 100.0}}}) + scope = manager._get_scope_manager("cost") + manager._apply_backoff({"cost": {"delay": 5.0, "consumed": 2.0}}) + assert scope._consumed == pytest.approx(2.0) + assert scope._backoff_level == 1 + + @coroutine_test + async def test_response_backoff_non_backoff_code(self): + manager = _manager() + assert await manager.get_response_backoff(_response(status=200)) is None + + @coroutine_test + async def test_response_backoff_429_without_header(self): + manager = _manager() + assert ( + await manager.get_response_backoff(_response(status=429)) == "example.com" + ) + + @pytest.mark.parametrize( + ("header", "expected_delay"), + [ + ({"Retry-After": "7"}, 7.0), + ({"RateLimit-Reset": "12"}, 12.0), + ], + ids=["retry-after", "ratelimit-reset"], + ) + @coroutine_test + async def test_response_backoff_delay_header(self, header, expected_delay): + manager = _manager() + data = await manager.get_response_backoff(_response(status=429, headers=header)) + assert data == {"example.com": {"delay": expected_delay}} + + @coroutine_test + async def test_response_backoff_retry_after_http_date(self): + manager = _manager() + # A date far in the past yields no positive delay. + data = await manager.get_response_backoff( + _response( + status=503, + headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}, + ) + ) + assert data == "example.com" + + @coroutine_test + async def test_response_backoff_max_of_both_headers(self): + manager = _manager() + data = await manager.get_response_backoff( + _response( + status=429, + headers={"Retry-After": "5", "RateLimit-Reset": "9"}, + ) + ) + assert data == {"example.com": {"delay": 9.0}} + + @coroutine_test + async def test_response_backoff_dont_track(self): + manager = _manager() + response = _response(status=429, meta={"dont_throttle": True}) + assert await manager.get_response_backoff(response) is None + + @pytest.mark.parametrize( + ("exception", "expected"), + [ + (DownloadTimeoutError(), "example.com"), + (ValueError(), None), + ], + ids=["tracked", "untracked"], + ) + @coroutine_test + async def test_exception_backoff(self, exception, expected): + manager = _manager() + request = Request("http://example.com") + assert await manager.get_exception_backoff(request, exception) == expected + + @coroutine_test + async def test_exception_backoff_dont_track(self): + manager = _manager() + request = Request("http://example.com", meta={"dont_throttle": True}) + assert ( + await manager.get_exception_backoff(request, DownloadTimeoutError()) is None + ) + + @pytest.mark.parametrize( + ("settings", "parser_delay", "expected_base_delay"), + [ + ({"ROBOTSTXT_OBEY": True, "RANDOMIZE_DOWNLOAD_DELAY": False}, 3.0, 3.0), + ({"ROBOTSTXT_OBEY": True, "THROTTLER_ROBOTSTXT_OBEY": False}, 3.0, 0.0), + ({"ROBOTSTXT_OBEY": True}, None, 0.0), + ({"ROBOTSTXT_OBEY": True}, ValueError(), 0.0), + ], + ids=["applies-delay", "obey-disabled", "no-delay", "backend-error"], + ) + def test_robots_parsed_signal(self, settings, parser_delay, expected_base_delay): + manager = _manager(settings) + manager.crawler.signals.send_catch_log( + signal=signals.robots_parsed, + robotparser=_FakeRobotParser(parser_delay), + request=Request("http://example.com/page"), + ) + scope = manager._get_scope_manager("example.com") + assert scope._base_delay == expected_base_delay + + def test_apply_robots_crawl_delay(self): + manager = _manager({"ROBOTSTXT_OBEY": True, "RANDOMIZE_DOWNLOAD_DELAY": False}) + manager.apply_robots_crawl_delay("example.com", 3.0) + scope = manager._get_scope_manager("example.com") + assert scope._base_delay == 3.0 + assert scope.can_send(now=0) == 0 # nothing sent yet + scope.record_sent(now=0) + assert scope.can_send(now=0) == pytest.approx(3.0) + + def test_apply_robots_crawl_delay_capped(self): + manager = _manager( + {"ROBOTSTXT_OBEY": True, "THROTTLER_ROBOTSTXT_MAX_DELAY": 2.0} + ) + manager.apply_robots_crawl_delay("example.com", 30.0) + assert manager._get_scope_manager("example.com")._base_delay == 2.0 + + def test_apply_robots_crawl_delay_disabled(self): + manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLER_ROBOTSTXT_OBEY": False}) + manager.apply_robots_crawl_delay("example.com", 3.0) + assert manager._get_scope_manager("example.com")._base_delay == 0.0 + + def test_apply_robots_crawl_delay_sets_concurrency(self): + manager = _manager({"ROBOTSTXT_OBEY": True}) + manager.apply_robots_crawl_delay("example.com", 3.0) + assert manager._get_scope_manager("example.com")._concurrency == 1 + + def test_apply_robots_crawl_delay_warns_on_conflict(self, caplog): + manager = _manager( + { + "ROBOTSTXT_OBEY": True, + "THROTTLER_SCOPES": {"example.com": {"concurrency": 8}}, + } + ) + with caplog.at_level(logging.WARNING, logger="scrapy.throttler"): + manager.apply_robots_crawl_delay("example.com", 3.0) + assert "Crawl-delay" in caplog.text + # The configured value takes precedence (crawl-delay not applied). + assert manager._get_scope_manager("example.com")._base_delay == 0.0 + + def test_apply_robots_crawl_delay_ignored(self, caplog): + manager = _manager( + { + "ROBOTSTXT_OBEY": True, + "THROTTLER_SCOPES": { + "example.com": {"concurrency": 8, "ignore_robots_txt": True} + }, + } + ) + with caplog.at_level(logging.WARNING, logger="scrapy.throttler"): + manager.apply_robots_crawl_delay("example.com", 3.0) + # No warning is logged and the crawl-delay is not applied. + assert "Crawl-delay" not in caplog.text + assert manager._get_scope_manager("example.com")._base_delay == 0.0 + + def test_scope_eviction(self): + manager = _manager({"THROTTLER_SCOPE_MAX_IDLE": 100.0}) + scope = manager._get_scope_manager("example.com") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # Not idle yet. + manager._last_eviction = None + manager._maybe_evict(now=50.0) + assert "example.com" in manager._scope_managers + # Idle past the threshold. + manager._last_eviction = None + manager._maybe_evict(now=201.0) + assert "example.com" not in manager._scope_managers + + def test_scope_eviction_skips_active_backoff(self): + manager = _manager( + {"THROTTLER_SCOPE_MAX_IDLE": 100.0, "BACKOFF_MAX_DELAY": 100_000.0} + ) + scope = manager._get_scope_manager("example.com") + scope.record_backoff(delay=10_000.0, now=0.0) + manager._last_eviction = None + manager._maybe_evict(now=5_000.0) + # Still in backoff (in_backoff_until far in the future), so not evicted + # even though it has been idle for longer than THROTTLER_SCOPE_MAX_IDLE. + assert "example.com" in manager._scope_managers + + def test_reserve_evicts_idle_scopes(self): + # A throttler-aware scheduler reserves every request before the engine + # reaches acquire() (which fast-paths reserved requests), so reserve() + # must be the hook that evicts idle scope managers; otherwise they pile + # up unbounded on broad crawls. + manager = _manager({"THROTTLER_SCOPE_MAX_IDLE": 1.0}) + idle = manager._get_scope_manager("idle.example") + # Make it look long-idle: a finished send in the distant monotonic past. + idle.record_sent(now=0.0) + idle.record_done(now=0.0) + assert "idle.example" in manager._scope_managers + manager.reserve(Request("http://active.example/1")) + assert "idle.example" not in manager._scope_managers + assert "active.example" in manager._scope_managers + + def test_scope_limit_evicts_least_recently_used(self): + manager = _manager({"THROTTLER_SCOPE_LIMIT": 2}) + # Use three scopes in order; each send/done leaves them idle. + for scope_id in ("a.example", "b.example", "c.example"): + scope = manager._get_scope_manager(scope_id) + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # The limit caps live managers at 2, dropping the least-recently-used. + assert set(manager._scope_managers) == {"b.example", "c.example"} + + def test_scope_limit_keeps_active_scopes(self): + manager = _manager({"THROTTLER_SCOPE_LIMIT": 1}) + # Two scopes with in-flight requests cannot be evicted, so the limit is + # exceeded rather than dropping a scope that still tracks a live send. + for scope_id in ("a.example", "b.example"): + manager._get_scope_manager(scope_id).record_sent(now=0.0) + assert set(manager._scope_managers) == {"a.example", "b.example"} + + def test_scope_limit_disabled(self): + manager = _manager({"THROTTLER_SCOPE_LIMIT": 0}) + for i in range(5): + scope = manager._get_scope_manager(f"{i}.example") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + assert len(manager._scope_managers) == 5 + + +class TestThrottlerScopeManager: + def test_no_delay_by_default(self): + scope = _scope_manager() + scope.record_sent(now=0.0) + assert scope.can_send(now=0.0) == 0 + + def test_base_delay_enforced(self): + scope = _scope_manager( + {"RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x", "delay": 2.0} + ) + scope.record_sent(now=10.0) + assert scope.can_send(now=10.0) == pytest.approx(2.0) + assert scope.can_send(now=11.0) == pytest.approx(1.0) + assert scope.can_send(now=12.0) == 0 + + def test_base_delay_defaults_to_download_delay(self): + # With no explicit scope "delay", the base delay is DOWNLOAD_DELAY. + scope = _scope_manager( + {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x"} + ) + assert scope._base_delay == pytest.approx(2.0) + + def test_scope_delay_overrides_download_delay(self): + # An explicit scope "delay" overrides DOWNLOAD_DELAY. + scope = _scope_manager( + {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, + {"id": "x", "delay": 0.0}, + ) + assert scope._base_delay == pytest.approx(0.0) + + def test_exponential_backoff(self): + scope = _scope_manager( + { + "BACKOFF_MIN_DELAY": 1.0, + "BACKOFF_DELAY_FACTOR": 2.0, + "BACKOFF_JITTER": 0, + }, + ) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(1.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(2.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(4.0) + + def test_backoff_cap(self): + scope = _scope_manager( + { + "BACKOFF_MIN_DELAY": 1.0, + "BACKOFF_DELAY_FACTOR": 10.0, + "BACKOFF_MAX_DELAY": 5.0, + "BACKOFF_JITTER": 0, + }, + ) + for _ in range(5): + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(5.0) + + @pytest.mark.parametrize( + ("max_delay", "backoff_delay", "expected"), + [ + (100.0, 20.0, 20.0), + (10.0, 999.0, 10.0), + ], + ids=["within-cap", "capped"], + ) + def test_retry_after_delay(self, max_delay, backoff_delay, expected): + scope = _scope_manager({"BACKOFF_MAX_DELAY": max_delay}) + scope.record_backoff(delay=backoff_delay, now=0.0) + assert scope.can_send(now=0.0) == pytest.approx(expected) + + def test_uncapped_backoff_delay(self): + # cap=False (used by trusted delay_scope() calls) ignores BACKOFF_MAX_DELAY. + scope = _scope_manager({"BACKOFF_MAX_DELAY": 10.0}) + scope.record_backoff(delay=999.0, now=0.0, cap=False) + assert scope.can_send(now=0.0) == pytest.approx(999.0) + + def test_recovery_after_window(self): + scope = _scope_manager( + { + "BACKOFF_MIN_DELAY": 1.0, + "BACKOFF_DELAY_FACTOR": 2.0, + "BACKOFF_WINDOW": 60.0, + "BACKOFF_JITTER": 0, + }, + ) + scope.record_backoff(now=0.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(2.0) + # One window passes with no new backoff -> one step down. + scope.can_send(now=60.0) + assert scope._backoff_level == 1 + assert scope._delay == pytest.approx(1.0) + # Another window -> back to base (0). + scope.can_send(now=120.0) + assert scope._backoff_level == 0 + assert scope._delay == pytest.approx(0.0) + + def test_per_scope_backoff_override(self): + scope = _scope_manager( + {"BACKOFF_MIN_DELAY": 1.0, "BACKOFF_DELAY_FACTOR": 2.0}, + { + "id": "x", + "backoff": {"min_delay": 5.0, "delay_factor": 3.0, "jitter": 0}, + }, + ) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(5.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(15.0) + + def test_set_base_delay_raises_only(self): + scope = _scope_manager( + {"RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x", "delay": 5.0} + ) + scope.set_base_delay(2.0) # lower -> ignored + assert scope._base_delay == 5.0 + scope.set_base_delay(8.0) # higher -> applied + assert scope._base_delay == 8.0 + assert scope._delay == 8.0 + + def test_min_delay_first_step(self): + scope = _scope_manager( + {"BACKOFF_MIN_DELAY": 3.0, "BACKOFF_DELAY_FACTOR": 2.0, "BACKOFF_JITTER": 0} + ) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(3.0) + + def test_default_scope_concurrency(self): + scope = _scope_manager() + assert scope._concurrency == 8 + + def test_no_scope_concurrency_limit_when_zero(self): + # THROTTLER_SCOPE_CONCURRENCY governs scopes that are neither a domain + # nor an IP (here a bare "custom" group name). + scope = _scope_manager( + settings={"THROTTLER_SCOPE_CONCURRENCY": 0}, config={"id": "custom"} + ) + assert scope._concurrency is None + for _ in range(100): + scope.record_sent(now=0.0) + assert scope.can_send(now=0.0) == 0 + assert scope.concurrency_blocked() is False + + def test_concurrency_limit(self): + scope = _scope_manager(config={"id": "x", "concurrency": 2}) + scope.record_sent(now=0.0) + # Concurrency is enforced via concurrency_blocked(), not can_send(). + assert scope.can_send(now=0.0) == 0 + assert scope.concurrency_blocked() is False + scope.record_sent(now=0.0) + # Two in flight, limit reached -> blocked. + assert scope.can_send(now=0.0) == 0 + assert scope.concurrency_blocked() is True + scope.record_done(now=0.0) + assert scope.concurrency_blocked() is False + + def test_record_done_fires_slot_event(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + scope.record_sent(now=0.0) + event = scope.slot_event() + assert not event.called + scope.record_done(now=0.0) + assert event.called + + def test_zero_backoff_window_recovers_at_once(self): + # A non-positive window must not make _recover spin forever; it recovers + # fully on the next can_send() instead. + scope = _scope_manager(settings={"BACKOFF_WINDOW": 0}, config={"id": "x"}) + scope.record_backoff(now=0.0) + assert scope._backoff_level == 1 + scope.can_send(now=10.0) + assert scope._backoff_level == 0 + assert scope._delay == scope._base_delay + + def test_zero_quota_window_keeps_quota_reset(self): + # A non-positive quota window must not make _maybe_reset_quota spin + # forever; the quota stays continuously reset instead. + scope = _scope_manager(config={"id": "x", "quota": 10.0, "window": 0}) + scope.record_sent(now=0.0, amount=10.0) + assert scope.can_send(now=1.0, amount=5.0) == 0.0 + assert scope._consumed == 0.0 + + def test_set_concurrency_fires_slot_event(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + scope.record_sent(now=0.0) + event = scope.slot_event() + assert not event.called + scope.set_concurrency(5) + assert event.called + + def test_discard_slot_event(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + event = scope.slot_event() + scope.discard_slot_event(event) + scope.discard_slot_event(event) # idempotent + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + assert not event.called + + def test_fire_slot_waiters_skips_already_fired(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + scope.record_sent(now=0.0) + event = scope.slot_event() + event.callback(None) # fired out-of-band before the slot frees up + # record_done() fires the waiters; the already-fired one is skipped + # rather than called a second time (which would raise). + scope.record_done(now=0.0) + assert event.called + + def test_set_concurrency_respects_min(self): + scope = _scope_manager(config={"id": "x", "min_concurrency": 3}) + scope.set_concurrency(1) + assert scope._concurrency == 3 + scope.set_concurrency(5) + assert scope._concurrency == 5 + + def test_quota_blocks_when_exhausted(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0, "window": 60.0}) + scope.record_sent(now=0.0, amount=6.0) + assert scope.can_send(now=0.0, amount=3.0) == 0 # 9 <= 10 + scope.record_sent(now=0.0, amount=3.0) + # 9 spent; a 3.0 request would exceed the quota -> wait for the window. + assert scope.can_send(now=0.0, amount=3.0) == pytest.approx(60.0) + # The window resets and quota is available again. + assert scope.can_send(now=60.0, amount=3.0) == 0 + + def test_quota_allows_oversized_request(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + # A single request larger than the whole quota is still allowed. + assert scope.can_send(now=0.0, amount=999.0) == 0 + + def test_quota_reconcile_consumed_delta(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + scope.record_sent(now=0.0, amount=2.0) + assert scope._consumed == pytest.approx(2.0) + # The response reports it actually consumed 0.5 more than estimated. + scope.reconcile_quota(consumed=0.5, now=0.0) + assert scope._consumed == pytest.approx(2.5) + + def test_quota_reconcile_remaining(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + scope.record_sent(now=0.0, amount=2.0) + scope.reconcile_quota(remaining=3.0, now=0.0) + assert scope._consumed == pytest.approx(7.0) + + +class TestThrottlerReadiness: + """The synchronous readiness API used by a throttler-aware scheduler.""" + + @coroutine_test + async def test_is_ready_unconstrained_scope(self): + manager = _manager() + request = Request("http://example.com/a") + # A scope with no configured delay/concurrency/quota is always ready. + assert await manager.get_scopes(request) == "example.com" + assert manager.is_ready(request) is True + + @coroutine_test + async def test_is_ready_without_cached_scopes(self): + # is_ready falls back to synchronous resolution when get_scopes was not + # called first (e.g. for a request restored from disk). + manager = _manager() + request = Request("http://example.com/a") + assert manager.is_ready(request) is True + + @coroutine_test + async def test_reserve_blocks_delay_scope(self): + manager = _manager( + { + "THROTTLER_SCOPES": {"example.com": {"delay": 100.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + } + ) + first = Request("http://example.com/1") + second = Request("http://example.com/2") + await manager.get_scopes(first) + await manager.get_scopes(second) + assert manager.is_ready(first) is True + manager.reserve(first) + # The base delay now blocks any further request for the scope. + assert manager.is_ready(second) is False + assert manager.get_time_until_ready(second) == pytest.approx(100.0, abs=1.0) + + @coroutine_test + async def test_delay_blocks_until_deadline(self): + manager = _manager({"THROTTLER_DEBUG": True}) + request = Request("http://example.com/a", meta={"delay": 100.0}) + await manager.get_scopes(request) + # The per-request delay holds back the request even though its scope is + # otherwise unconstrained. + assert manager.is_ready(request) is False + assert manager.get_time_until_ready(request) == pytest.approx(100.0, abs=1.0) + # The deadline is computed once and reused by later polls. + deadline = request.meta["_throttler_delay_deadline"] + assert manager.is_ready(request) is False + assert request.meta["_throttler_delay_deadline"] == deadline + + @coroutine_test + async def test_delay_not_reapplied_once_consumed(self): + # A request whose delay was already honored (e.g. promoted out of a + # throttler-aware queue's holding area, or restored on resume) is + # ready, so it cannot re-block its scope set on a stale deadline. + manager = _manager() + request = Request( + "http://example.com/a", + meta={"delay": 100.0, "_throttler_delayed": True}, + ) + await manager.get_scopes(request) + assert manager.is_ready(request) is True + assert manager.get_request_delay(request) == 0.0 + + @coroutine_test + async def test_get_request_delay(self): + manager = _manager() + assert manager.get_request_delay( + Request("http://example.com/a", meta={"delay": 100.0}) + ) == pytest.approx(100.0, abs=1.0) + # A request without a per-request delay is not held individually. + assert manager.get_request_delay(Request("http://example.com/b")) == 0.0 + + @coroutine_test + async def test_delay_scope(self): + manager = _manager({"THROTTLER_DEBUG": True, "RANDOMIZE_DOWNLOAD_DELAY": False}) + request = Request("http://example.com/a") + await manager.get_scopes(request) + assert manager.is_ready(request) is True + # A component can delay a whole scope on demand, like a Retry-After + # response header does. + manager.delay_scope("example.com", 50.0) + assert manager.is_ready(request) is False + assert manager.get_time_until_ready(request) == pytest.approx(50.0, abs=1.0) + + @coroutine_test + async def test_delay_scope_bypasses_max_delay(self): + # BACKOFF_MAX_DELAY caps untrusted input (headers), but delay_scope is a + # trusted call, so it may exceed the cap. + manager = _manager( + {"BACKOFF_MAX_DELAY": 30.0, "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + manager.delay_scope("example.com", 1000.0) + assert manager.get_time_until_ready(request) == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_reserve_blocks_on_concurrency(self): + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}}) + first = Request("http://example.com/1") + second = Request("http://example.com/2") + await manager.get_scopes(first) + await manager.get_scopes(second) + manager.reserve(first) + assert manager.is_ready(second) is False + # Pure concurrency blocking is not time-gated. + assert manager.get_time_until_ready(second) is None + manager.release(first) + assert manager.is_ready(second) is True + + @coroutine_test + async def test_acquire_noop_when_reserved(self): + manager = _manager( + { + "THROTTLER_SCOPES": {"example.com": {"delay": 100.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + } + ) + request = Request("http://example.com/1") + await manager.get_scopes(request) + manager.reserve(request) + # A reserved request fast-paths through acquire() without re-recording + # the send or waiting for the delay. + await manager.acquire(request) + scope = manager._get_scope_manager("example.com") + assert scope._active == 1 # reserve recorded exactly one send + + @coroutine_test + async def test_get_scope_load(self): + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 4}}}) + assert manager.get_scope_load("example.com") == 0.0 + request = Request("http://example.com/1") + await manager.get_scopes(request) + manager.reserve(request) + assert manager.get_scope_load("example.com") == pytest.approx(0.25) + + def test_get_scope_load_falls_back_to_global_concurrency(self): + manager = _manager({"CONCURRENT_REQUESTS": 8}) + # A scope with no explicit concurrency limit uses CONCURRENT_REQUESTS. + request = Request("http://example.com/1") + manager.reserve(request) + assert manager.get_scope_load("example.com") == pytest.approx(1 / 8) + + +class TestParseRateHeaders: + @pytest.mark.parametrize( + ("value", "expected"), + [ + (b"\xff\xfe", None), # undecodable bytes + ("garbage-not-a-date", None), # neither a number nor a valid date + # A naive HTTP date (no timezone) in the past is treated as UTC and + # yields no positive delay. + ("Wed, 21 Oct 2015 07:28:00", None), + ], + ids=["undecodable", "garbage", "naive-past-date"], + ) + def test_parse_retry_after(self, value, expected): + assert _parse_retry_after(_response(headers={"Retry-After": value})) == expected + + @pytest.mark.parametrize( + "value", + [b"\xff\xfe", "not-a-number"], + ids=["undecodable", "non-numeric"], + ) + def test_parse_ratelimit_reset_invalid(self, value): + assert ( + _parse_ratelimit_reset(_response(headers={"RateLimit-Reset": value})) + is None + ) + + +class TestScopeHelpers: + def test_iter_scopes(self): + assert list(iter_scopes(None)) == [] + assert list(iter_scopes("a")) == ["a"] + assert list(iter_scopes({"a": 1.0, "b": 2.0})) == ["a", "b"] + assert list(iter_scopes(["a", "b"])) == ["a", "b"] + + def test_iter_scope_values(self): + assert list(iter_scope_values(None)) == [] + assert list(iter_scope_values("a")) == [("a", None)] + assert list(iter_scope_values({"a": 1.0})) == [("a", 1.0)] + assert list(iter_scope_values(["a", "b"])) == [("a", None), ("b", None)] + + def test_to_scope_dict(self): + assert _to_scope_dict({"a": 1}, list) == {"a": 1} + assert _to_scope_dict(None, list) == {} + assert _to_scope_dict("a", lambda: None) == {"a": None} + assert _to_scope_dict(["a", "b"], dict) == {"a": {}, "b": {}} + with pytest.raises(TypeError): + _to_scope_dict(123, dict) + + def test_add_bare_scope(self): + assert _add_bare_scope(None, "a", None) == "a" + assert _add_bare_scope("a", "a", None) == "a" + assert _add_bare_scope("a", "b", None) == {"a", "b"} + assert _add_bare_scope({"a": 1}, "b", {}) == {"a": 1, "b": {}} + assert _add_bare_scope({"a": 1}, "a", {}) == {"a": 1} + assert _add_bare_scope(["a"], "b", None) == {"a", "b"} + assert _add_bare_scope(["a"], "a", None) == ["a"] + with pytest.raises(TypeError): + _add_bare_scope(123, "a", None) + + def test_add_scope_without_value(self): + assert add_scope(None, "a") == "a" + assert add_scope("a", "b") == {"a", "b"} + + def test_add_scope_with_value(self): + assert add_scope(None, "a", 2.0) == {"a": 2.0} + assert add_scope({"a": None}, "b", 3.0) == {"a": None, "b": 3.0} + + def test_add_scope_with_value_rejects_non_dict_entry(self): + with pytest.raises(TypeError): + add_scope({"a": 1.0}, "a", 2.0) + + def test_update_scope_backoff_without_params(self): + assert update_scope_backoff(None, "a") == "a" + assert update_scope_backoff("a", "b") == {"a", "b"} + + def test_update_scope_backoff_with_params(self): + assert update_scope_backoff(None, "a", delay=5.0, consumed=2.0) == { + "a": {"delay": 5.0, "consumed": 2.0} + } + assert update_scope_backoff({"a": {}}, "a", delay=1.0) == {"a": {"delay": 1.0}} + assert update_scope_backoff(None, "a", consumed=2.0) == {"a": {"consumed": 2.0}} + + def test_update_scope_backoff_rejects_non_dict_entry(self): + with pytest.raises(TypeError): + update_scope_backoff({"a": 1.0}, "a", delay=1.0) + + +class TestThrottlerEdges: + @coroutine_test + async def test_acquire_without_scopes(self): + manager = _manager() + request = Request("http://example.com/a", meta={"throttler_scopes": []}) + # No scopes resolve, so acquire() returns without reserving anything. + await manager.acquire(request) + assert request not in manager._reserved + + def test_get_scope_load_without_concurrency_limit(self): + manager = _manager({"CONCURRENT_REQUESTS": 0}) + # CONCURRENT_REQUESTS is 0, so the load denominator is 0 and the load is + # reported as 0 instead of raising. + assert manager.get_scope_load("example.com") == 0.0 + + @coroutine_test + async def test_acquire_logs_and_waits_for_delay(self): + manager = _manager( + { + "THROTTLER_SCOPES": {"example.com": {"delay": 0.02}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "THROTTLER_DEBUG": True, + } + ) + r1 = Request("http://example.com/1") + r2 = Request("http://example.com/2") + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r1))) + # r2 must wait out the per-scope delay accrued by r1 before it proceeds. + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r2))) + assert r2 in manager._reserved + + @coroutine_test + async def test_acquire_logs_while_waiting_for_slot(self): + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + + manager = _manager( + { + "THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}, + "THROTTLER_DEBUG": True, + } + ) + r1 = Request("http://example.com/1") + r2 = Request("http://example.com/2") + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r1))) + call_later(0, manager.release, r1) + # r2 is concurrency-blocked and waits, with debug logging, for the slot. + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r2))) + assert r2 in manager._reserved + + @coroutine_test + async def test_delay_request(self): + manager = _manager({"THROTTLER_DEBUG": True}) + request = Request("http://example.com/a", meta={"delay": 0.01}) + await manager._delay_request(request) + assert request.meta["_throttler_delayed"] is True + # A second call is a no-op (the request was already delayed). + await manager._delay_request(request) + + @coroutine_test + async def test_delay_request_without_debug(self): + # Same as above but with debug logging off, so the delay is applied + # without emitting the debug message. + manager = _manager() + request = Request("http://example.com/a", meta={"delay": 0.01}) + await manager._delay_request(request) + assert request.meta["_throttler_delayed"] is True + + @coroutine_test + async def test_wait_for_slot_discards_unfired_events(self): + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + + manager = _manager() + m1 = _scope_manager(config={"id": "a", "concurrency": 1}) + m2 = _scope_manager(config={"id": "b", "concurrency": 1}) + m1.record_sent(now=0.0) + m2.record_sent(now=0.0) + # Free m1's slot on the next tick so the wait wakes up with m1's event + # fired while m2's event is still pending. + call_later(0, m1.record_done) + await manager._wait_for_slot([m1, m2]) + # The still-pending m2 event is discarded from its waiter list. + assert m2._slot_waiters == [] + + @coroutine_test + async def test_process_exception_applies_backoff(self): + manager = _manager() + request = Request("http://example.com/a") + await manager.process_exception(request, DownloadTimeoutError()) + assert manager._get_scope_manager("example.com")._backoff_level == 1 + + def test_apply_backoff_debug_logging(self, caplog): + manager = _manager({"THROTTLER_DEBUG": True}) + with caplog.at_level(logging.DEBUG, logger="scrapy.throttler"): + manager._apply_backoff("example.com") + assert "Backoff for scope" in caplog.text + assert manager._get_scope_manager("example.com")._backoff_level == 1 + + def test_on_robots_parsed_disabled(self): + manager = _manager({"THROTTLER_ROBOTSTXT_OBEY": False}) + # With obeying disabled, the handler returns without touching any scope. + manager._on_robots_parsed(_FakeRobotParser(5.0), Request("http://example.com")) + assert not manager._scope_managers + + def test_apply_robots_crawl_delay_warns_on_delay_conflict(self, caplog): + manager = _manager( + { + "ROBOTSTXT_OBEY": True, + "THROTTLER_SCOPES": {"example.com": {"delay": 0.5}}, + } + ) + with caplog.at_level(logging.WARNING, logger="scrapy.throttler"): + manager.apply_robots_crawl_delay("example.com", 3.0) + assert "Crawl-delay" in caplog.text + + def test_apply_robots_crawl_delay_debug_logging(self, caplog): + manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLER_DEBUG": True}) + with caplog.at_level(logging.DEBUG, logger="scrapy.throttler"): + manager.apply_robots_crawl_delay("example.com", 3.0) + assert "Crawl-delay" in caplog.text + + def test_maybe_evict_disabled(self): + manager = _manager({"THROTTLER_SCOPE_MAX_IDLE": 0}) + scope = manager._get_scope_manager("example.com") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # Eviction disabled (max idle <= 0): the scope is kept regardless. + manager._maybe_evict(now=10_000.0) + assert "example.com" in manager._scope_managers + + +class TestThrottlerScopeManagerEdges: + def test_jitter_as_range(self): + scope = _scope_manager( + config={"id": "x", "backoff": {"jitter": [0.0, 0.0]}, "min_delay": 1.0} + ) + # A [low, high] jitter range of [0, 0] leaves the value unchanged. + assert scope._apply_jitter(4.0, scope._backoff_jitter) == pytest.approx(4.0) + + def test_effective_delay_randomized(self): + scope = _scope_manager( + {"RANDOMIZE_DOWNLOAD_DELAY": True}, {"id": "x", "delay": 2.0} + ) + scope.record_sent(now=0.0) + # A randomized base delay lands within [0.5, 1.5] * delay. + assert 1.0 <= scope.can_send(now=0.0) <= 3.0 + + def test_record_done_without_active(self): + scope = _scope_manager(config={"id": "x"}) + # Calling record_done() with nothing in flight is a harmless no-op. + scope.record_done(now=0.0) + assert scope._active == 0 + + def test_reconcile_quota_no_change(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + scope.record_sent(now=0.0, amount=4.0) + # Neither consumed nor remaining given: the estimate is left untouched. + scope.reconcile_quota(now=0.0) + assert scope._consumed == pytest.approx(4.0) + + def test_set_base_delay_during_backoff(self): + scope = _scope_manager( + {"BACKOFF_MIN_DELAY": 1.0, "BACKOFF_JITTER": 0}, {"id": "x"} + ) + scope.record_backoff(now=0.0) + backoff_delay = scope._delay + # Raising the base delay mid-backoff updates the base but not the current + # (higher) backoff delay. + scope.set_base_delay(0.5) + assert scope._base_delay == 0.5 + assert scope._delay == backoff_delay + + def test_record_sent_clears_expired_backoff(self): + scope = _scope_manager(config={"id": "x"}) + scope.record_backoff(delay=5.0, now=0.0) + assert scope._in_backoff_until == pytest.approx(5.0) + scope.record_sent(now=10.0) + # The hard backoff window has passed, so it is cleared. + assert scope._in_backoff_until is None + + def test_reconcile_quota_without_quota(self): + scope = _scope_manager(config={"id": "x"}) + # No quota configured, so reconciliation is a no-op. + scope.reconcile_quota(consumed=5.0, now=0.0) + assert scope._consumed == 0.0 + + def test_is_idle_with_active_requests(self): + scope = _scope_manager(config={"id": "x"}) + scope.record_sent(now=0.0) + # An in-flight request keeps the scope from being evicted. + assert scope.is_idle(now=10_000.0, max_idle=1.0) is False + + def test_is_idle_when_never_used(self): + scope = _scope_manager(config={"id": "x"}) + # A scope that was never used (no last_seen) is idle. + assert scope.is_idle(now=0.0, max_idle=1.0) is True + + +class TestThrottlerIntegration: + @coroutine_test + async def test_backoff_recorded_on_429(self, mockserver): + crawler = get_crawler(SimpleSpider, {"RETRY_ENABLED": False}) + await crawler.crawl_async( + mockserver.url("/status?n=429"), mockserver=mockserver + ) + throttler = crawler.throttler + assert throttler is not None + managers = throttler._scope_managers + assert managers, "no throttler scope was created" + assert any(manager._backoff_level >= 1 for manager in managers.values()) + + @coroutine_test + async def test_backoff_recorded_on_download_error(self, mockserver): + crawler = get_crawler(SimpleSpider, {"RETRY_ENABLED": False}) + # A dropped connection raises a DownloadFailedError, which the engine + # routes through the throttler before re-raising. + await crawler.crawl_async( + mockserver.url("/drop?abort=1"), mockserver=mockserver + ) + throttler = crawler.throttler + assert throttler is not None + managers = throttler._scope_managers + assert managers, "no throttler scope was created" + assert any(manager._backoff_level >= 1 for manager in managers.values()) + + @coroutine_test + async def test_no_backoff_on_200(self, mockserver): + crawler = get_crawler(SimpleSpider) + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) + throttler = crawler.throttler + assert throttler is not None + assert all( + manager._backoff_level == 0 + for manager in throttler._scope_managers.values() + ) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a198d1c09..cd3f44433 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -13,6 +13,8 @@ from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, is_asyncio_available, + sleep, + wait_for_first, ) from tests.utils.decorators import coroutine_test @@ -158,3 +160,42 @@ class TestAsyncioLoopingCall: looping_call.start(0.1) assert not looping_call.running assert "Error calling the AsyncioLoopingCall function" in caplog.text + + +class TestSleep: + @coroutine_test + async def test_sleep(self): + # A zero-second sleep completes without error under any reactor mode. + await sleep(0) + + +class TestWaitForFirst: + @coroutine_test + async def test_empty(self): + # No deferreds -> two empty sets, returned immediately. + assert await wait_for_first([]) == (set(), set()) + + @coroutine_test + async def test_returns_done_deferred(self): + fired: Deferred[None] = Deferred() + fired.callback(None) + done, pending = await wait_for_first([fired], timeout=1) + assert done == {fired} + assert pending == set() + + @coroutine_test + async def test_no_timeout(self): + # timeout=None -> no timeout deferred is created. + fired: Deferred[None] = Deferred() + fired.callback(None) + done, pending = await wait_for_first([fired]) + assert done == {fired} + assert pending == set() + + @coroutine_test + async def test_timeout_elapses(self): + # When the timeout fires first, every input deferred is still pending. + never: Deferred[None] = Deferred() + done, pending = await wait_for_first([never], timeout=0.01) + assert done == set() + assert pending == {never} diff --git a/tox.ini b/tox.ini index 5017d7636..f643caa46 100644 --- a/tox.ini +++ b/tox.ini @@ -144,7 +144,7 @@ deps = lxml==4.6.4 parsel==1.5.0 pyOpenSSL==22.0.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 w3lib==1.17.0 zope.interface==5.1.0 @@ -260,7 +260,7 @@ deps = lxml==5.3.2 parsel==1.5.0 pyOpenSSL==24.3.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses # an inline flag placement that Python 3.11 treats as an error: global