From 44b03539df1c985e3a0df3f0b58ff466d66eb24d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sat, 28 Jun 2025 14:32:56 +0200 Subject: [PATCH 01/55] WIP --- docs/index.rst | 5 + docs/topics/settings.rst | 41 ---- docs/topics/throttling.rst | 481 +++++++++++++++++++++++++++++++++++++ 3 files changed, 486 insertions(+), 41 deletions(-) create mode 100644 docs/topics/throttling.rst diff --git a/docs/index.rst b/docs/index.rst index 1a9cf636c..f36986bb8 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -166,6 +166,7 @@ Solving specific problems topics/jobs topics/coroutines topics/asyncio + topics/throttling :doc:`faq` Get answers to most frequently asked questions. @@ -212,6 +213,10 @@ Solving specific problems :doc:`topics/asyncio` Use :mod:`asyncio` and :mod:`asyncio`-powered libraries. +:doc:`topics/throttling` + Control request throttling to avoid overloading websites and comply with + rate limits. + .. _extending-scrapy: Extending Scrapy diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 2a1be5f88..11d1dec5c 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -502,47 +502,6 @@ Default: ``100`` Maximum number of concurrent items (per response) to process in parallel in :ref:`item pipelines `. -.. setting:: CONCURRENT_REQUESTS - -CONCURRENT_REQUESTS -------------------- - -Default: ``16`` - -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. - -.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - -CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------- - -Default: ``8`` - -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed to any single domain. - -See also: :ref:`topics-autothrottle` and its -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. - - -.. setting:: CONCURRENT_REQUESTS_PER_IP - -CONCURRENT_REQUESTS_PER_IP --------------------------- - -Default: ``0`` - -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed to any single IP. If non-zero, the -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is -used instead. In other words, concurrency limits will be applied per IP, not -per domain. - -This setting also affects :setting:`DOWNLOAD_DELAY` and -:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP` -is non-zero, download delay is enforced per IP, not per domain. - .. setting:: DEFAULT_DROPITEM_LOG_LEVEL DEFAULT_DROPITEM_LOG_LEVEL diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst new file mode 100644 index 000000000..b0d40d213 --- /dev/null +++ b/docs/topics/throttling.rst @@ -0,0 +1,481 @@ +.. _throttling: + +========== +Throttling +========== + +Scrapy provides several mechanisms to control the rate at which requests are +sent, to prevent website overloading and handle rate limiting responses. + +Basic throttling +================ + +The main :ref:`settings ` to control throttling are: + +- :setting:`CONCURRENT_REQUESTS`: The maximum number of total concurrent + requests. + +- :setting:`THROTTLING_BUCKET_CONCURRENCY`: The maximum number of concurrent + requests per :ref:`throttling bucket `. + +- :setting:`THROTTLING_BUCKET_DELAY`: The minimum seconds to wait between + consecutive requests to the same :ref:`throttling bucket + `. + +.. + TODO: Add a section about the handling of subdomains. By default, Scrapy + should treat subdomains as separate slots, but it should be easy to change + that behavior for specific domains, maybe with some new setting. + +.. _throttling-buckets: + +Throttling buckets +================== + +.. + TODO: Cover what they are, what their default values are, how to change + them easily on a per-request base. + + +Settings +======== + +.. setting:: CONCURRENT_REQUESTS + +CONCURRENT_REQUESTS +------------------- + +Default: ``16`` + +The maximum number of concurrent (i.e. simultaneous) requests that will be +performed by the Scrapy downloader. + +.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN + +CONCURRENT_REQUESTS_PER_DOMAIN +------------------------------ + +Default: ``8`` + +The maximum number of concurrent (i.e. simultaneous) requests that will be +performed to any single domain. + +See also: :ref:`topics-autothrottle` and its +:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. + + + +Global Request Limits +====================== + +The simplest form of throttling is limiting the total number of concurrent requests across your entire spider. + +CONCURRENT_REQUESTS +------------------- + +The :setting:`CONCURRENT_REQUESTS` setting controls the maximum number of requests that can be processed simultaneously across all domains: + +.. code-block:: python + + # settings.py + CONCURRENT_REQUESTS = 16 # Default value + +This is a global limit that affects all requests regardless of their target domain. Setting it to a lower value will make your spider more conservative and polite, but slower. Setting it higher will make requests faster but may overwhelm servers or your own network connection. + +**When to use**: This is your first line of defense against sending too many requests at once. Good default values are typically between 8-32 depending on your needs and the target servers' capacity. + +Per-Domain Request Limits +========================== + +More sophisticated throttling involves limiting requests on a per-domain basis, which is usually more appropriate since different servers have different capacities. + +CONCURRENT_REQUESTS_PER_DOMAIN +------------------------------- + +The :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting limits concurrent requests to each individual domain: + +.. code-block:: python + + # settings.py + CONCURRENT_REQUESTS_PER_DOMAIN = 8 # Default value + +This means that even if you have :setting:`CONCURRENT_REQUESTS` set to 32, no single domain will receive more than 8 concurrent requests. This prevents one fast-responding domain from monopolizing all your concurrent request slots. + +**Example**: If you're scraping both ``example.com`` and ``other-site.com``, each domain will be limited to 8 concurrent requests, allowing a total of up to 16 concurrent requests (but still subject to the global :setting:`CONCURRENT_REQUESTS` limit). + +CONCURRENT_REQUESTS_PER_IP +--------------------------- + +Similar to per-domain limits, :setting:`CONCURRENT_REQUESTS_PER_IP` limits concurrent requests per IP address: + +.. code-block:: python + + # settings.py + CONCURRENT_REQUESTS_PER_IP = 1 # Default value + +This is useful when multiple domains resolve to the same IP address (common with CDNs or shared hosting). The IP-based limit takes precedence over the domain-based limit when they conflict. + +DOWNLOAD_DELAY +-------------- + +While concurrency limits control how many requests are sent simultaneously, :setting:`DOWNLOAD_DELAY` controls the time delay between requests to the same domain: + +.. code-block:: python + + # settings.py + DOWNLOAD_DELAY = 3 # Wait 3 seconds between requests to the same domain + +This setting introduces a delay between consecutive requests to the same domain/slot. It's applied per-domain, so requests to different domains are not affected by each other's delays. + +The delay can be randomized using :setting:`RANDOMIZE_DOWNLOAD_DELAY`: + +.. code-block:: python + + # settings.py + DOWNLOAD_DELAY = 3 + RANDOMIZE_DOWNLOAD_DELAY = True # Default + # This will use delays between 1.5 and 4.5 seconds (0.5 * DOWNLOAD_DELAY to 1.5 * DOWNLOAD_DELAY) + +**When to use**: Download delays are particularly useful for websites that are sensitive to request frequency but can handle multiple concurrent connections. They're also helpful for mimicking human-like browsing patterns. + +**Note**: A global download delay doesn't make sense because it would unnecessarily slow down requests to different domains. The per-domain approach allows you to be respectful to each server individually while maintaining overall efficiency. + +Custom Download Slots +====================== + +For more advanced scenarios, you can customize how requests are grouped for throttling purposes using download slots. + +Understanding Download Slots +----------------------------- + +By default, Scrapy groups requests by their domain name for throttling purposes. Each domain gets its own "download slot" with its own concurrency limits and delays. However, you can customize this grouping: + +.. code-block:: python + + # In your spider + def start_requests(self): + # Default: requests to example.com go to "example.com" slot + yield scrapy.Request("https://example.com/page1") + yield scrapy.Request("https://example.com/page2") + + # Custom: group these requests differently + yield scrapy.Request( + "https://api.example.com/fast", meta={"download_slot": "api_fast"} + ) + yield scrapy.Request( + "https://api.example.com/slow", meta={"download_slot": "api_slow"} + ) + +Per-Slot Configuration +---------------------- + +You can configure different throttling settings for different slots using :setting:`DOWNLOAD_SLOTS`: + +.. code-block:: python + + # settings.py + DOWNLOAD_SLOTS = { + "api_fast": { + "concurrency": 1, + "delay": 0.5, + }, + "api_slow": { + "concurrency": 1, + "delay": 5.0, + }, + "images": { + "concurrency": 8, + "delay": 0, + }, + } + +**Use cases**: +- Different API endpoints with different rate limits +- Separating expensive operations (like browser rendering) from cheap ones +- Treating different subdomains with different politeness levels +- Grouping requests by authentication context + +Advanced Throttling System +=========================== + +For complex scraping scenarios that require fine-grained control over throttling, Scrapy provides an advanced throttling bucket system that goes beyond simple per-domain limits. + +Throttling Buckets +------------------ + +The new throttling system introduces the concept of "throttling buckets" - resources that requests consume and that can become temporarily unavailable when limits are exceeded. Unlike download slots, a single request can require multiple buckets, enabling multi-dimensional throttling. + +Enabling Throttling Buckets +---------------------------- + +.. code-block:: python + + # settings.py + THROTTLING_ENABLED = True + THROTTLING_BUCKET_MANAGER = "scrapy.throttling.DefaultBucketManager" + +Basic Bucket Usage +------------------ + +The simplest bucket configuration replicates domain-based throttling: + +.. code-block:: python + + # Custom bucket manager + class MyBucketManager: + def get_request_buckets(self, request, spider): + domain = urlparse(request.url).netloc + return {domain: 1.0} # Consume 1 unit of the domain bucket + + def process_response(self, response, request, spider): + if response.status == 429: # Too Many Requests + domain = urlparse(request.url).netloc + # Throttle this domain for 60 seconds + self.throttle_bucket(domain, delay=60) + +Multi-Dimensional Throttling +----------------------------- + +The real power comes from using multiple buckets per request: + +.. code-block:: python + + def get_request_buckets(self, request, spider): + buckets = {} + + # Domain-based throttling + domain = urlparse(request.url).netloc + buckets[domain] = 1.0 + + # API feature-based throttling + if "browser=true" in request.url: + buckets["browser_rendering"] = 1.0 + + if "extract=true" in request.url: + buckets["ai_extraction"] = 5.0 # More expensive + + # Geographic throttling + if "region=eu" in request.url: + buckets["eu_datacenter"] = 1.0 + + return buckets + +Cost-Based Throttling +--------------------- + +Some APIs charge different amounts for different operations. The bucket system supports fractional consumption: + +.. code-block:: python + + def get_request_buckets(self, request, spider): + buckets = {"api_credits": 1.0} # Default cost + + if "operation=expensive" in request.url: + buckets["api_credits"] = 10.0 # Costs 10x more + + return buckets + + + def process_response(self, response, request, spider): + # Update actual consumption based on response + if "X-Actual-Cost" in response.headers: + actual_cost = float(response.headers["X-Actual-Cost"]) + # Could update bucket consumption here for better accuracy + +Responding to Server Signals +----------------------------- + +The bucket system can respond intelligently to server throttling signals: + +.. code-block:: python + + def process_response(self, response, request, spider): + if response.status == 429: + # Check for Retry-After header + retry_after = response.headers.get("Retry-After") + if retry_after: + delay = int(retry_after) + else: + delay = 60 # Default backoff + + # Determine which bucket to throttle based on response + if "rate limit exceeded for API key" in response.text: + self.throttle_bucket("api_key_global", delay=delay) + elif "too many requests to this endpoint" in response.text: + endpoint = self._extract_endpoint(request.url) + self.throttle_bucket(f"endpoint_{endpoint}", delay=delay) + + elif response.status == 503: + # Service unavailable - throttle the entire domain + domain = urlparse(request.url).netloc + self.throttle_bucket(domain, delay=300) # 5 minutes + +Configuration +------------- + +The throttling system provides several configuration options: + +.. code-block:: python + + # settings.py + THROTTLING_ENABLED = True + THROTTLING_BUCKET_MANAGER = "myproject.throttling.CustomBucketManager" + + # Maximum number of delayed requests to keep in memory + THROTTLING_MAX_DELAYED_REQUESTS = 1000 + + # Warn when delayed requests exceed this threshold + THROTTLING_DELAYED_REQUESTS_WARN_THRESHOLD = 500 + +Integration with Direct Downloads +--------------------------------- + +The throttling system also works with direct downloads made via ``crawler.engine.download()``: + +.. code-block:: python + + # In a pipeline or extension + @inlineCallbacks + def process_item(self, item, spider): + # This request will respect throttling buckets + request = scrapy.Request(item["image_url"]) + response = yield spider.crawler.engine.download(request) + # Process response... + +Best Practices +============== + +Choosing the Right Approach +---------------------------- + +1. **Start Simple**: Begin with :setting:`CONCURRENT_REQUESTS` and :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` +2. **Add Delays When Needed**: Use :setting:`DOWNLOAD_DELAY` for frequency-sensitive sites +3. **Use Custom Slots for Special Cases**: When different parts of a site need different treatment +4. **Advanced Buckets for Complex APIs**: When dealing with modern APIs with sophisticated rate limiting + +Respectful Scraping +------------------- + +- Always check ``robots.txt`` and respect ``Crawl-delay`` directives +- Monitor server response times and adjust settings if you're causing delays +- Watch for 429, 503, and other error responses that indicate you're going too fast +- Consider the server's perspective: your efficiency shouldn't come at their expense + +Common Patterns +--------------- + +**E-commerce Site**: + +.. code-block:: python + + CONCURRENT_REQUESTS_PER_DOMAIN = 2 + DOWNLOAD_DELAY = 1 + RANDOMIZE_DOWNLOAD_DELAY = True + +**REST API**: + +.. code-block:: python + + # Use throttling buckets to respect API rate limits + THROTTLING_ENABLED = True + CONCURRENT_REQUESTS_PER_DOMAIN = 5 + +**Mixed Content (API + Web)**: + +.. code-block:: python + + DOWNLOAD_SLOTS = { + "api": {"concurrency": 10, "delay": 0.1}, + "web": {"concurrency": 2, "delay": 2.0}, + } + +Monitoring and Debugging +======================== + +Scrapy provides several ways to monitor your throttling: + +.. code-block:: python + + # Enable autothrottle debugging (for legacy autothrottle extension only) + AUTOTHROTTLE_DEBUG = True + + # Monitor stats + # Check spider.crawler.stats for throttling-related statistics + +.. warning:: + The AutoThrottle extension is deprecated and not recommended for new projects. It uses a simplistic latency-based approach that doesn't align with modern server throttling patterns. Use the throttling bucket system instead. + +Settings Reference +================== + +Global Settings +--------------- + +.. setting:: CONCURRENT_REQUESTS + +**Default**: ``16`` + +The maximum number of concurrent requests performed by Scrapy. + +.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN + +**Default**: ``8`` + +The maximum number of concurrent requests performed to any single domain. + +.. setting:: CONCURRENT_REQUESTS_PER_IP + +**Default**: ``1`` + +The maximum number of concurrent requests performed to any single IP address. + +.. setting:: DOWNLOAD_DELAY + +**Default**: ``0`` + +The amount of time (in seconds) that the downloader should wait before downloading consecutive pages from the same domain. + +.. setting:: RANDOMIZE_DOWNLOAD_DELAY + +**Default**: ``True`` + +If enabled, Scrapy will wait a random amount of time (between 0.5 * and 1.5 * ``DOWNLOAD_DELAY``) while fetching requests from the same domain. + +Slot Settings +------------- + +.. setting:: DOWNLOAD_SLOTS + +**Default**: ``{}`` + +A dictionary containing the download slots and their settings. Each slot can have the following settings: + +* ``concurrency`` - Maximum concurrent requests for this slot +* ``delay`` - Download delay for this slot (in seconds) + +Advanced Throttling Settings +---------------------------- + +.. setting:: THROTTLING_ENABLED + +**Default**: ``False`` + +Enable the advanced throttling bucket system. + +.. setting:: THROTTLING_BUCKET_MANAGER + +**Default**: ``'scrapy.throttling.DefaultBucketManager'`` + +A string specifying the throttling bucket manager to use. + +.. setting:: THROTTLING_MAX_DELAYED_REQUESTS + +**Default**: ``1000`` + +Maximum number of delayed requests to keep in memory. + +.. setting:: THROTTLING_DELAYED_REQUESTS_WARN_THRESHOLD + +**Default**: ``500`` + +Warn when the number of delayed requests exceeds this threshold. From 07fd9265d0a78e3ab51114fe406d27b70b87b9f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 29 Jun 2025 02:44:00 +0200 Subject: [PATCH 02/55] WIP --- docs/topics/components.rst | 51 +-- docs/topics/downloader-middleware.rst | 16 + docs/topics/request-response.rst | 1 + docs/topics/settings.rst | 67 --- docs/topics/throttling.rst | 631 ++++++++++++++++++++++---- scrapy/throttling.py | 81 ++++ 6 files changed, 654 insertions(+), 193 deletions(-) create mode 100644 scrapy/throttling.py diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 56f8c6498..617c668c2 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,39 +9,24 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`ADDONS` - -- :setting:`DNS_RESOLVER` - -- :setting:`DOWNLOAD_HANDLERS` - -- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` - -- :setting:`DOWNLOADER_MIDDLEWARES` - -- :setting:`DUPEFILTER_CLASS` - -- :setting:`EXTENSIONS` - -- :setting:`FEED_EXPORTERS` - -- :setting:`FEED_STORAGES` - -- :setting:`ITEM_PIPELINES` - -- :setting:`SCHEDULER` - -- :setting:`SCHEDULER_DISK_QUEUE` - -- :setting:`SCHEDULER_MEMORY_QUEUE` - -- :setting:`SCHEDULER_PRIORITY_QUEUE` - -- :setting:`SCHEDULER_START_DISK_QUEUE` - -- :setting:`SCHEDULER_START_MEMORY_QUEUE` - -- :setting:`SPIDER_MIDDLEWARES` +- :setting:`ADDONS` +- :setting:`DNS_RESOLVER` +- :setting:`DOWNLOAD_HANDLERS` +- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` +- :setting:`DOWNLOADER_MIDDLEWARES` +- :setting:`DUPEFILTER_CLASS` +- :setting:`EXTENSIONS` +- :setting:`FEED_EXPORTERS` +- :setting:`FEED_STORAGES` +- :setting:`ITEM_PIPELINES` +- :setting:`SCHEDULER` +- :setting:`SCHEDULER_DISK_QUEUE` +- :setting:`SCHEDULER_MEMORY_QUEUE` +- :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` +- :setting:`SCHEDULER_START_MEMORY_QUEUE` +- :setting:`SPIDER_MIDDLEWARES` +- :setting:`THROTTLING_MANAGER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 60b6aab78..6bd58e27e 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -886,6 +886,22 @@ Default: ``20`` The maximum number of redirections that will be followed for a single request. If maximum redirections are exceeded, the request is aborted and ignored. +.. setting:: REDIRECT_MAX_DELAY + +REDIRECT_MAX_DELAY +^^^^^^^^^^^^^^^^^^ + +Default: ``120.0`` + +If a redirect response provides a ``Retry-After`` header, the redirect is only +followed after the specified delay. This setting caps that delay to a specific +number of seconds. Set to ``0.0`` to disable these delays altogether. + +Note that redirect delays only affect the request being redirected, they do not +delay other requests to the same domain. ``Retry-After`` only does that during +:ref:`backoff `. + + MetaRefreshMiddleware --------------------- diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8a907e377..122022bdb 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -652,6 +652,7 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`throttling_scopes` .. reqmeta:: bindaddress diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 11d1dec5c..3e8de9107 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -823,49 +823,6 @@ Default: ``True`` Whether to enable downloader stats collection. -.. setting:: DOWNLOAD_DELAY - -DOWNLOAD_DELAY --------------- - -Default: ``0`` - -Minimum seconds to wait between 2 consecutive requests to the same domain. - -Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting -servers too hard. - -Decimal numbers are supported. For example, to send a maximum of 4 requests -every 10 seconds:: - - DOWNLOAD_DELAY = 2.5 - -This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting, which is enabled by default. - -When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced -per IP address instead of per domain. - -Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain -concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response -time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective -concurrency for that domain is 1. When testing throttling configurations, it -usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first, -and only increase :setting:`DOWNLOAD_DELAY` once -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is -desired. - -.. _spider-download_delay-attribute: - -.. note:: - - This delay can be set per spider using :attr:`download_delay` spider attribute. - -It is also possible to change this setting per domain, although it requires -non-trivial code. See the implementation of the :ref:`AutoThrottle -` extension for an example. - - .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS @@ -950,30 +907,6 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2: .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 -.. setting:: DOWNLOAD_SLOTS - -DOWNLOAD_SLOTS --------------- - -Default: ``{}`` - -Allows to define concurrency/delay parameters on per slot (domain) basis: - - .. code-block:: python - - DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, - "books.toscrape.com": {"delay": 3, "randomize_delay": False}, - } - -.. note:: - - For other downloader slots default settings values will be used: - - - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` - .. setting:: DOWNLOAD_TIMEOUT diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index b0d40d213..ce4d5ee00 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -4,32 +4,462 @@ Throttling ========== -Scrapy provides several mechanisms to control the rate at which requests are -sent, to prevent website overloading and handle rate limiting responses. +Sending too many requests too quickly can `overload websites`_. To avoid that, +you must throttle_ your requests. Scrapy can :ref:`throttle requests +`, :ref:`handle backoff `, and much more. -Basic throttling -================ +.. _overload websites: https://en.wikipedia.org/wiki/Denial-of-service_attack +.. _throttle: https://en.wikipedia.org/wiki/Bandwidth_throttling -The main :ref:`settings ` to control throttling are: +.. _basic-throttling: -- :setting:`CONCURRENT_REQUESTS`: The maximum number of total concurrent - requests. +Concurrency and delay +===================== -- :setting:`THROTTLING_BUCKET_CONCURRENCY`: The maximum number of concurrent - requests per :ref:`throttling bucket `. +Use the following :ref:`settings ` to configure the default +throttling for each :ref:`throttling scope `: -- :setting:`THROTTLING_BUCKET_DELAY`: The minimum seconds to wait between - consecutive requests to the same :ref:`throttling bucket - `. +- .. setting:: THROTTLING_CONCURRENCY + + **THROTTLING_CONCURRENCY** (default: ``1``) + + The maximum number of concurrent requests. + +- .. setting:: THROTTLING_DELAY + + **THROTTLING_DELAY** (default: ``1.0``) + + The minimum seconds to wait between consecutive requests. + +- .. setting:: THROTTLING_JITTER + + **THROTTLING_JITTER** (default: ``0.5``, i.e. ±50%) + + Randomize delays by this factor, i.e. the final delay is a random value + between ``delay*(1-jitter)`` and ``delay*(1+jitter)``. + + It can be set to a 2-item list with low and high factors, e.g. + ``[-0.1, 0.3]`` to randomize delays between ``delay*0.9`` and + ``delay*1.3``. + +.. setting:: THROTTLING_SCOPES + +Use **THROTTLING_SCOPES** (default: ``{}``) to override these values for +specific throttling scopes: + + .. code-block:: python + + THROTTLING_SCOPES = { + "books.toscrape.com": {"concurrency": 16, "delay": 0.0}, + "example.com": {"jitter": 0.2}, + } + +:setting:`THROTTLING_SCOPES` can also :ref:`override backoff settings +`. + +When setting these values, note that: + +- :setting:`CONCURRENT_REQUESTS` effectively caps concurrency for any + throttling scope. + +- When higher than response time, delay effectively limits concurrency to + ``1``. + + +.. _crawl-delay: + +Crawl-Delay directive +--------------------- + +`Crawl-Delay `__ +is a non-standard ``robots.txt`` directive that indicates a number of seconds to +wait between requests. + +.. setting:: THROTTLING_ROBOTSTXT_OBEY +.. setting:: THROTTLING_ROBOTSTXT_MAX_DELAY + +If :setting:`ROBOTSTXT_OBEY` and **THROTTLING_ROBOTSTXT_OBEY** are +``True`` (default), valid ``Crawl-Delay`` directives override +:setting:`THROTTLING_CONCURRENCY` and :setting:`THROTTLING_DELAY`. Concurrency +is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at +**THROTTLING_ROBOTSTXT_MAX_DELAY** (default: ``60.0``). + +If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it +will be respected, but a warning will be logged about the discrepancy with +``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. + + +.. _backoff: + +Backoff +======= + +When a response or network error warrants backoff, `exponential backoff`_ is +used to reduce request rate. + +.. _exponential backoff: https://en.wikipedia.org/wiki/Exponential_backoff + +In such cases, every new request with the same throttling scope is sent with +its delay multiplied by some factor (up to some maximum) or set to some minimum +value (if it was lower), until a request gets a response that does not require +backoff. Once a response that does not require backoff is received, the delay +is gradually reduced back to its original value. + +The following settings control backoff behavior: + +- .. setting:: BACKOFF_HTTP_CODES + + **BACKOFF_HTTP_CODES** (default: ``[429, 502, 503, 504, 520, 521, 522, 523, 524]``) + + HTTP response status codes that warrant backoff. + + Usually, all codes here should be in :setting:`RETRY_HTTP_CODES` as well, + but not all codes in :setting:`RETRY_HTTP_CODES` need to be here: some bad + responses may require a retry without backoff. + +- .. setting:: BACKOFF_EXCEPTIONS + + **BACKOFF_EXCEPTIONS** + + Default: + + .. code-block:: python + + [ + "twisted.internet.defer.TimeoutError", + "twisted.internet.error.TimeoutError", + "twisted.internet.error.TCPTimedOutError", + "twisted.web.client.ResponseFailed", + ] + + Exception classes that warrant backoff. Strings are interpreted as import + paths. + + Usually, all exceptions here should be in :setting:`RETRY_EXCEPTIONS` as + well, but not all exceptions in :setting:`RETRY_EXCEPTIONS` need to be + here: some errors may require a retry without backoff. + +- .. setting:: BACKOFF_FACTOR + + **BACKOFF_FACTOR** (default: ``2.0``) + + The factor by which the delay is multiplied for each new request sent to a + given throttling scope during backoff. + +- .. setting:: BACKOFF_MAX_DELAY + + **BACKOFF_MAX_DELAY** (default: ``300.0``) + + The maximum delay that can be applied during backoff. If the delay exceeds + this value, it will be capped at this value. + +- .. setting:: BACKOFF_MIN_DELAY + + **BACKOFF_MIN_DELAY** (default: ``1.0``) + + The minimum delay that can be applied during backoff. If the delay is less + than this value, it will be set to this value. Must be higher than ``0.0``. + +- .. setting:: BACKOFF_JITTER + + **BACKOFF_JITTER** (default: ``0.1``) + + Overrides :setting:`THROTTLING_JITTER` during backoff. + +When a throttling scope is configured with a **concurrency higher than 1**, +backoff is handled separately per concurrency slot. If at some point all +concurrency slots reach the maximum backoff delay, a “concurrency backoff” +starts, controlled by the following setting: + +- .. setting:: BACKOFF_CONCURRENCY_DECREASE_FACTOR + + **BACKOFF_CONCURRENCY_DECREASE_FACTOR** (default: ``0.5``) + + The factor by which the concurrency is decreased during concurrency + backoff. + +.. _scope-backoff: + +Backoff settings can be overridden per throttling scope using +:setting:`THROTTLING_SCOPES`: + +.. code-block:: python + + { + "example.com": { + "backoff": { + "http_codes": [429, 503], + "exceptions": ["builtins.IOError"], + "factor": 1.2, + "max_delay": 180.0, + "min_delay": 5.0, + "jitter": [0.01, 0.33], + "concurrency_decrease_factor": 0.8, + } + }, + } + + +.. _retry-after: + +Rate limiting headers +--------------------- + +The `Retry-After +`__ +and the `RateLimit-Reset +`__ +HTTP response headers indicate how long to wait before making a follow-up +request. + +They are taken into account during :ref:`backoff `: their value is +read (the highest if both headers are present), capped at +:setting:`BACKOFF_MAX_DELAY`, and used as a minimum delay, i.e. it is used if +higher than the current delay but ignored if lower. + +.. seealso:: :setting:`REDIRECT_MAX_DELAY` + + +.. _throttling-scopes: + +Scopes +====== + +Throttling scopes represent aspects of requests that can be throttled +independently. + +.. + For future reference, the “throttling scope” name was taken from + https://www.ietf.org/archive/id/draft-polli-ratelimit-headers-02.html#section-1.4-4.4 + +.. _default-throttling-scopes: + +Default throttling scopes +-------------------------- + +By default, each request has a single throttling scope representing the domain +or subdomain of the target URL. That is, when you set the concurrency or delay +of a throttling scope, it applies to all requests made to that domain or +subdomain. + +For example, https://books.toscrape.com and +https://books.toscrape.com/catalogue/page-2.html both get a +``books.toscrape.com`` throttling scope. + +Note however that subdomains are treated as separate throttling scopes by +default. For example, https://toscrape.com gets a ``toscrape.com`` throttling +scope, and ``books.toscrape.com`` and ``toscrape.com`` are considered +unrelated throttling scopes. If you want to change this behavior, see +:ref:`alternative-domain-throttling`. + + +.. _custom-throttling-scopes: + +Customizing throttling scopes +------------------------------ + +There are 2 ways to customize throttling scopes. + +.. reqmeta:: throttling_scopes + +For simple use cases, you can use the ``throttling_scopes`` request metadata +key: + +.. code-block:: python + + Request("https://example.com/", meta={"throttling_scopes": "foo"}) + Request("https://example.com/", meta={"throttling_scopes": {"foo", "bar"}}) + Request("https://example.com/", meta={"throttling_scopes": {"foo": 1.0, "bar": 2.5}}) + +.. note:: Throttling scopes set through request metadata remain through the + request lifetime, e.g. throught redirects, even if those change the request + URL. + +.. setting:: THROTTLING_MANAGER + +For anything else, set **THROTTLING_MANAGER** (default: +:class:`~scrapy.throttling.ThrottlingManager`) to a :ref:`component +` that implements the +:class:`~scrapy.throttling.ThrottlingManagerProtocol` protocol (or its import +path as a string): + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_MANAGER = "myproject.throttling.MyThrottlingManager" + + +.. _throttling-quotas: + +Quotas +====== + +When different requests can consume different amounts of a throttling scope, +you can express this using quotas. + +In the :reqmeta:`throttling_scopes` request metadata key and in the +:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scopes` method you use +a :class:`dict` structure where keys are throttling scopes and values are +:class:`float` that indicate the amount of a scope that the request is expected +to consume (it does not need to be exact). + +By default, those values are ignored. However, if a call to +:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_response_throttling` or +:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_exception_throttling` +reports available quotas for one or more throttling scopes, request quotas will +start being tracked and determine which requests can be sent and which cannot. + +In fact, everything else being equal, +:class:`~scrapy.pqueues.ScrapyPriorityQueue` prioritizes requests that consume +a higher portion of the available quota, to minimize the risk of those +requests getting stuck. + + +API +=== + +.. autoclass:: scrapy.throttling.ThrottlingManagerProtocol + :members: + :member-order: bysource + +.. autoclass:: scrapy.throttling.ThrottlingManager + + +Examples +======== + +.. _alternative-domain-throttling: + +Alternative domain throttling +----------------------------- + +If you are not happy with the :ref:`default throttling scope behavior +` with regards to domains and subdomains, you can +change it. + +Alternative approaches include: + +- Using the **highest-level registrable domain** as the throttling scope, + e.g. https://books.toscrape.com and https://toscrape.com both get a + ``toscrape.com`` throttling scope. + + This allows to apply the same throttling settings to all subdomains of a + registrable domain. + + For example: + + .. code-block:: python + :caption: ``settings.py`` + + import tldextract + from scrapy.utils.httpobj import urlparse_cached + + + class MyThrottlingManager: + + def get_request_scopes(self, request): + extracted = tldextract.extract(request.url) + if extracted.domain and extracted.suffix: + return f"{extracted.domain}.{extracted.suffix}" + return urlparse_cached(request).netloc + + + THROTTLING_MANAGER = MyThrottlingManager + +- Using **multiple throttling scopes per request**, one per registrable + domain and for every higher-level subdomain, e.g. + https://books.toscrape.com and https://toscrape.com both get a + ``toscrape.com`` throttling scope, but https://books.toscrape.com also + gets a ``books.toscrape.com`` throttling scope. + + This allows to apply the same throttling settings to all subdomains of a + registrable domain, but also allows applying further restrictions on each + or on some subdomains. + + For example: + + .. code-block:: python + :caption: ``settings.py`` + + import tldextract + from scrapy.utils.httpobj import urlparse_cached + + + class MyThrottlingManager: + + def get_request_scopes(self, request): + extracted = tldextract.extract(request.url) + if not (extracted.domain and extracted.suffix): + return urlparse_cached(request).netloc + scopes = set() + registrable_domain = f"{extracted.domain}.{extracted.suffix}" + scopes.add(registrable_domain) + if extracted.subdomain: + subdomain_parts = extracted.subdomain.split(".") + for i in range(len(subdomain_parts)): + subdomain = ".".join(subdomain_parts[i:]) + full_domain = f"{subdomain}.{registrable_domain}" + scopes.add(full_domain) + return scopes + + + THROTTLING_MANAGER = MyThrottlingManager + THROTTLING_SCOPES = { + "toscrape.com": {"concurrency": 32}, + "books.toscrape.com": {"concurrency": 24}, + "quotes.toscrape.com": {"concurrency": 16}, + } + + Here ``books.toscrape.com`` requests can reach 24 concurrency and + ``quotes.toscrape.com`` requests can reach 16 concurrency, but never both + at the same time, because that would sum 40 concurrency, and + ``toscrape.com`` requests are limited to 32. + + + + + + + + + + + +Requests can be assigned 1 or more throttling scopes, each with a value. + + + +This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` +setting, which is enabled by default. + +When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced +per IP address instead of per domain. + +Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain +concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response +time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective +concurrency for that domain is 1. When testing throttling configurations, it +usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first, +and only increase :setting:`DOWNLOAD_DELAY` once +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is +desired. + +.. _spider-download_delay-attribute: + +.. note:: + + This delay can be set per spider using :attr:`download_delay` spider attribute. + +It is also possible to change this setting per domain, although it requires +non-trivial code. See the implementation of the :ref:`AutoThrottle +` extension for an example. .. TODO: Add a section about the handling of subdomains. By default, Scrapy should treat subdomains as separate slots, but it should be easy to change that behavior for specific domains, maybe with some new setting. -.. _throttling-buckets: - -Throttling buckets +Throttling scopes ================== .. @@ -40,28 +470,30 @@ Throttling buckets Settings ======== -.. setting:: CONCURRENT_REQUESTS -CONCURRENT_REQUESTS -------------------- +.. setting:: DOWNLOAD_SLOTS -Default: ``16`` +DOWNLOAD_SLOTS +-------------- -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed by the Scrapy downloader. +Default: ``{}`` -.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN +Allows to define concurrency/delay parameters on per slot (domain) basis: -CONCURRENT_REQUESTS_PER_DOMAIN ------------------------------- + .. code-block:: python -Default: ``8`` + DOWNLOAD_SLOTS = { + "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, + "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + } -The maximum number of concurrent (i.e. simultaneous) requests that will be -performed to any single domain. +.. note:: -See also: :ref:`topics-autothrottle` and its -:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option. + For other downloader slots default settings values will be used: + + - :setting:`DOWNLOAD_DELAY`: ``delay`` + - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` + - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` @@ -198,12 +630,12 @@ You can configure different throttling settings for different slots using :setti Advanced Throttling System =========================== -For complex scraping scenarios that require fine-grained control over throttling, Scrapy provides an advanced throttling bucket system that goes beyond simple per-domain limits. +For complex scraping scenarios that require fine-grained control over throttling, Scrapy provides an advanced throttling scope system that goes beyond simple per-domain limits. Throttling Buckets ------------------ -The new throttling system introduces the concept of "throttling buckets" - resources that requests consume and that can become temporarily unavailable when limits are exceeded. Unlike download slots, a single request can require multiple buckets, enabling multi-dimensional throttling. +The new throttling system introduces the concept of "throttling scopes" - resources that requests consume and that can become temporarily unavailable when limits are exceeded. Unlike download slots, a single request can require multiple scopes, enabling multi-dimensional throttling. Enabling Throttling Buckets ---------------------------- @@ -212,80 +644,80 @@ Enabling Throttling Buckets # settings.py THROTTLING_ENABLED = True - THROTTLING_BUCKET_MANAGER = "scrapy.throttling.DefaultBucketManager" + THROTTLING_MANAGER = "scrapy.throttling.DefaultBucketManager" Basic Bucket Usage ------------------ -The simplest bucket configuration replicates domain-based throttling: +The simplest scope configuration replicates domain-based throttling: .. code-block:: python - # Custom bucket manager + # Custom scope manager class MyBucketManager: - def get_request_buckets(self, request, spider): + def get_request_scopes(self, request, spider): domain = urlparse(request.url).netloc - return {domain: 1.0} # Consume 1 unit of the domain bucket + return {domain: 1.0} # Consume 1 unit of the domain scope def process_response(self, response, request, spider): if response.status == 429: # Too Many Requests domain = urlparse(request.url).netloc # Throttle this domain for 60 seconds - self.throttle_bucket(domain, delay=60) + self.throttle_scope(domain, delay=60) Multi-Dimensional Throttling ----------------------------- -The real power comes from using multiple buckets per request: +The real power comes from using multiple scopes per request: .. code-block:: python - def get_request_buckets(self, request, spider): - buckets = {} + def get_request_scopes(self, request, spider): + scopes = {} # Domain-based throttling domain = urlparse(request.url).netloc - buckets[domain] = 1.0 + scopes[domain] = 1.0 # API feature-based throttling if "browser=true" in request.url: - buckets["browser_rendering"] = 1.0 + scopes["browser_rendering"] = 1.0 if "extract=true" in request.url: - buckets["ai_extraction"] = 5.0 # More expensive + scopes["ai_extraction"] = 5.0 # More expensive # Geographic throttling if "region=eu" in request.url: - buckets["eu_datacenter"] = 1.0 + scopes["eu_datacenter"] = 1.0 - return buckets + return scopes Cost-Based Throttling --------------------- -Some APIs charge different amounts for different operations. The bucket system supports fractional consumption: +Some APIs charge different amounts for different operations. The scope system supports fractional consumption: .. code-block:: python - def get_request_buckets(self, request, spider): - buckets = {"api_credits": 1.0} # Default cost + def get_request_scopes(self, request, spider): + scopes = {"api_credits": 1.0} # Default cost if "operation=expensive" in request.url: - buckets["api_credits"] = 10.0 # Costs 10x more + scopes["api_credits"] = 10.0 # Costs 10x more - return buckets + return scopes def process_response(self, response, request, spider): # Update actual consumption based on response if "X-Actual-Cost" in response.headers: actual_cost = float(response.headers["X-Actual-Cost"]) - # Could update bucket consumption here for better accuracy + # Could update scope consumption here for better accuracy Responding to Server Signals ----------------------------- -The bucket system can respond intelligently to server throttling signals: +The scope system can respond intelligently to server throttling signals: .. code-block:: python @@ -298,17 +730,17 @@ The bucket system can respond intelligently to server throttling signals: else: delay = 60 # Default backoff - # Determine which bucket to throttle based on response + # Determine which scope to throttle based on response if "rate limit exceeded for API key" in response.text: - self.throttle_bucket("api_key_global", delay=delay) + self.throttle_scope("api_key_global", delay=delay) elif "too many requests to this endpoint" in response.text: endpoint = self._extract_endpoint(request.url) - self.throttle_bucket(f"endpoint_{endpoint}", delay=delay) + self.throttle_scope(f"endpoint_{endpoint}", delay=delay) elif response.status == 503: # Service unavailable - throttle the entire domain domain = urlparse(request.url).netloc - self.throttle_bucket(domain, delay=300) # 5 minutes + self.throttle_scope(domain, delay=300) # 5 minutes Configuration ------------- @@ -319,7 +751,7 @@ The throttling system provides several configuration options: # settings.py THROTTLING_ENABLED = True - THROTTLING_BUCKET_MANAGER = "myproject.throttling.CustomBucketManager" + THROTTLING_MANAGER = "myproject.throttling.CustomBucketManager" # Maximum number of delayed requests to keep in memory THROTTLING_MAX_DELAYED_REQUESTS = 1000 @@ -337,7 +769,7 @@ The throttling system also works with direct downloads made via ``crawler.engine # In a pipeline or extension @inlineCallbacks def process_item(self, item, spider): - # This request will respect throttling buckets + # This request will respect throttling scopes request = scrapy.Request(item["image_url"]) response = yield spider.crawler.engine.download(request) # Process response... @@ -376,7 +808,7 @@ Common Patterns .. code-block:: python - # Use throttling buckets to respect API rate limits + # Use throttling scopes to respect API rate limits THROTTLING_ENABLED = True CONCURRENT_REQUESTS_PER_DOMAIN = 5 @@ -403,7 +835,7 @@ Scrapy provides several ways to monitor your throttling: # Check spider.crawler.stats for throttling-related statistics .. warning:: - The AutoThrottle extension is deprecated and not recommended for new projects. It uses a simplistic latency-based approach that doesn't align with modern server throttling patterns. Use the throttling bucket system instead. + The AutoThrottle extension is deprecated and not recommended for new projects. It uses a simplistic latency-based approach that doesn't align with modern server throttling patterns. Use the throttling scope system instead. Settings Reference ================== @@ -411,31 +843,7 @@ Settings Reference Global Settings --------------- -.. setting:: CONCURRENT_REQUESTS - -**Default**: ``16`` - -The maximum number of concurrent requests performed by Scrapy. - -.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - -**Default**: ``8`` - -The maximum number of concurrent requests performed to any single domain. - -.. setting:: CONCURRENT_REQUESTS_PER_IP - -**Default**: ``1`` - -The maximum number of concurrent requests performed to any single IP address. - -.. setting:: DOWNLOAD_DELAY - -**Default**: ``0`` - -The amount of time (in seconds) that the downloader should wait before downloading consecutive pages from the same domain. - -.. setting:: RANDOMIZE_DOWNLOAD_DELAY +RANDOMIZE_DOWNLOAD_DELAY **Default**: ``True`` @@ -444,7 +852,7 @@ If enabled, Scrapy will wait a random amount of time (between 0.5 * and 1.5 * `` Slot Settings ------------- -.. setting:: DOWNLOAD_SLOTS +DOWNLOAD_SLOTS **Default**: ``{}`` @@ -460,13 +868,7 @@ Advanced Throttling Settings **Default**: ``False`` -Enable the advanced throttling bucket system. - -.. setting:: THROTTLING_BUCKET_MANAGER - -**Default**: ``'scrapy.throttling.DefaultBucketManager'`` - -A string specifying the throttling bucket manager to use. +Enable the advanced throttling scope system. .. setting:: THROTTLING_MAX_DELAYED_REQUESTS @@ -479,3 +881,46 @@ Maximum number of delayed requests to keep in memory. **Default**: ``500`` Warn when the number of delayed requests exceeds this threshold. + + + +.. + TODO: Provide real-life examples of throttling configurations, including + exception and error response handling, throttling based on responses, + delay adjustment geared towards rate limits optimization, querying of + external resources for throttling decisions, etc. + + + + +- .. setting:: CONCURRENT_REQUESTS + + ``CONCURRENT_REQUESTS`` (default: ``16``): The maximum number of total + concurrent requests. + +.. + TODO: Since this setting is more about limiting spider-side resources than + throttling, maybe it does not need to be covered in this page. + + +.. + Implement a signal that can be emitted to change the throttling config of + a specific throttling scope. + + +.. + TODO: Explain how things work, for every setting/parameter, when a request + is assigned multiple scopes. + +.. + TODO: Explain how scope units work, and give usage examples. + +.. + TODO: Support backoff in THROTTLING_SCOPES having a cls key with a class + or its import path as a string, and arbitraty kwargs to pass to its + __init__ method, to implement a custom backoff strategy. + + +.. + TODO: Make sure that the API is flexible enough to support scrapy-zyte-api + use cases. diff --git a/scrapy/throttling.py b/scrapy/throttling.py new file mode 100644 index 000000000..1d57e1a67 --- /dev/null +++ b/scrapy/throttling.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Protocol + +from scrapy.utils.httpobj import urlparse_cached + +if TYPE_CHECKING: + from collections.abc import Iterable + + from scrapy.http import Request, Response + + +class ThrottlingManagerProtocol(Protocol): + """A protocol for :setting:`THROTTLING_MANAGER` :ref:`components + `.""" + + def get_scopes( + self, request: Request + ) -> None | str | Iterable[str] | dict[str, float]: + """Return the :ref:`throttling scopes ` that apply + to *request*. + + Return ``None`` if no scopes apply, a string for a single scope, an + iterable of strings for multiple scopes, or a dict with scope names as + keys and :ref:`throttling quotas ` as values. + """ + + def get_response_throttling( + self, response: Response + ) -> None | str | Iterable[str] | dict[str, dict[str, Any]]: + """Return a throttling data update based on *response*. + + Return ``None`` if there is nothing new to report, i.e. the response is + not a :ref:`backoff ` response. + + If the response indicates that one or more scopes are currently + exhausted, return a string for a single scope or an iterable of strings + for multiple scopes. + + If the response indicates any other information about one or more + scopes, return a dict with scopes as keys and dict values. Dict values + support the following keys: + + - ``"delay"``: a float indicating how many seconds to wait before + sending another request for the scope. + + - ``"quota"``: a float indicating the remaining :ref:`throttling + quota `. + + If ``"quota"`` is not specified, the resource is considered exhausted. + + .. code-block:: python + + return { + "scope1": {"delay": 5.0}, + "scope2": {}, + "scope3": {"quota": 42.0}, + } + """ + + def get_exception_throttling( + self, request: Request, exception: Exception + ) -> None | str | Iterable[str] | dict[str, dict[str, Any]]: + """Return a throttling data update based on *exception* and the + *request* that caused it. + + It supports the same return values as :meth:`get_response_throttling`. + """ + + +class ThrottlingManager: + """The default :setting:`THROTTLING_MANAGER` class. + + It assigns to each request its domain or subdomain as scope and handles + backoff according to :ref:`backoff settings `. + """ + + def get_scopes( + self, request: Request + ) -> None | str | Iterable[str] | dict[str, float]: + return urlparse_cached(request).netloc From 3224b3669c274c9c73696cdfb51c880a9ccc9010 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Sun, 29 Jun 2025 23:00:45 +0200 Subject: [PATCH 03/55] WIP --- docs/conf.py | 12 + docs/index.rst | 12 +- docs/intro/overview.rst | 8 +- docs/news.rst | 12 +- docs/requirements.txt | 1 + docs/topics/autothrottle.rst | 195 --- docs/topics/request-response.rst | 1 - docs/topics/settings.rst | 32 +- docs/topics/throttling.rst | 1356 +++++++++-------- scrapy/crawler.py | 6 + scrapy/extensions/throttle.py | 11 +- .../templates/project/module/settings.py.tmpl | 84 +- scrapy/throttling.py | 375 ++++- 13 files changed, 1147 insertions(+), 958 deletions(-) delete mode 100644 docs/topics/autothrottle.rst diff --git a/docs/conf.py b/docs/conf.py index 493a62976..040229a4a 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,6 +34,7 @@ extensions = [ "sphinx.ext.coverage", "sphinx.ext.intersphinx", "sphinx.ext.viewcode", + "sphinx_reredirects", "sphinx_rtd_dark_mode", ] @@ -136,6 +137,12 @@ coverage_ignore_pyobjects = [ r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor", ] +# -- Options for the autodoc extension ---------------------------------------- +autodoc_type_aliases = { + "BackoffData": "BackoffData", + "GetScopesMethod": "GetScopesMethod", + "RequestScopes": "RequestScopes", +} # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration @@ -157,6 +164,11 @@ intersphinx_mapping = { } intersphinx_disabled_reftypes: Sequence[str] = [] +# -- sphinx-reredirects ------------------------------------------------------- +redirects = { + "topics/autothrottle": "throttling.html", +} + # -- Options for sphinx-hoverxref extension ---------------------------------- # https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html diff --git a/docs/index.rst b/docs/index.rst index f36986bb8..2343b853c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -161,12 +161,11 @@ Solving specific problems topics/leaks topics/media-pipeline topics/deploy - topics/autothrottle + topics/throttling topics/benchmarking topics/jobs topics/coroutines topics/asyncio - topics/throttling :doc:`faq` Get answers to most frequently asked questions. @@ -198,8 +197,9 @@ Solving specific problems :doc:`topics/deploy` Deploying your Scrapy spiders and run them in a remote server. -:doc:`topics/autothrottle` - Adjust crawl rate dynamically based on load. +:doc:`topics/throttling` + Control request throttling to avoid overloading websites and comply with + rate limits. :doc:`topics/benchmarking` Check how Scrapy performs on your hardware. @@ -213,10 +213,6 @@ Solving specific problems :doc:`topics/asyncio` Use :mod:`asyncio` and :mod:`asyncio`-powered libraries. -:doc:`topics/throttling` - Control request throttling to avoid overloading websites and comply with - rate limits. - .. _extending-scrapy: Extending Scrapy diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d05e46551..f62dca73a 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -81,11 +81,9 @@ error happens while handling it. While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you -control over the politeness of the crawl through :ref:`a few settings -`. You can do things like setting a download delay between -each request, limiting the amount of concurrent requests per domain or per IP, and -even :ref:`using an auto-throttling extension ` that tries -to figure these settings out automatically. +control over :ref:`throttling `, e.g. you can set a delay between +requests to the same domain, set a maximum concurrency per domain, or customize +the :ref:`backoff ` behavior. .. note:: diff --git a/docs/news.rst b/docs/news.rst index 7a235787e..b5dcfa487 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -969,10 +969,9 @@ New features :meth:`~scrapy.crawler.Crawler.get_spider_middleware`. (:issue:`6181`) -- Slot delay updates by the :ref:`AutoThrottle extension - ` based on response latencies can now be disabled for - specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta - key. +- Slot delay updates by the ``scrapy.extensions.throttle.AutoThrottle`` + extension based on response latencies can now be disabled for specific + requests via the ``autothrottle_dont_adjust_delay`` meta key. (:issue:`6246`, :issue:`6527`) - If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, @@ -4943,8 +4942,7 @@ The following deprecated APIs have been removed (:issue:`3578`): * From :class:`~scrapy.spiders.Spider` (and subclasses): - * ``DOWNLOAD_DELAY`` (use :ref:`download_delay - `) + * ``DOWNLOAD_DELAY`` (use ``download_delay``) * ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`) @@ -7121,7 +7119,7 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command -- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` +- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. diff --git a/docs/requirements.txt b/docs/requirements.txt index 103fb08d6..f828cdf9c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,6 @@ sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 +sphinx-reredirects==1.0.0 sphinx-rtd-theme==3.0.2 sphinx-rtd-dark-mode==1.3.0 diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst deleted file mode 100644 index 5bd72fa15..000000000 --- a/docs/topics/autothrottle.rst +++ /dev/null @@ -1,195 +0,0 @@ -.. _topics-autothrottle: - -====================== -AutoThrottle extension -====================== - -This is an extension for automatically throttling crawling speed based on load -of both the Scrapy server and the website you are crawling. - -Design goals -============ - -1. be nicer to sites instead of using default download delay of zero -2. automatically adjust Scrapy to the optimum crawling speed, so the user - doesn't have to tune the download delays to find the optimum one. - The user only needs to specify the maximum concurrent requests - it allows, and the extension does the rest. - -.. _autothrottle-algorithm: - -How it works -============ - -Scrapy allows defining the concurrency and delay of different download slots, -e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are -assigned to slots based on their URL domain, although it is possible to -customize the download slot of any request. - -The AutoThrottle extension adjusts the delay of each download slot dynamically, -to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent -requests on average to each remote website. - -It uses download latency to compute the delays. The main idea is the -following: if a server needs ``latency`` seconds to respond, a client -should send a request each ``latency/N`` seconds to have ``N`` requests -processed in parallel. - -Instead of adjusting the delays one can just set a small fixed -download delay and impose hard limits on concurrency using -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar -effect, but there are some important differences: - -* because the download delay is small there will be occasional bursts - of requests; -* often non-200 (error) responses can be returned faster than regular - responses, so with a small download delay and a hard concurrency limit - crawler will be sending requests to server faster when server starts to - return errors. But this is an opposite of what crawler should do - in case - of errors it makes more sense to slow down: these errors may be caused by - the high request rate. - -AutoThrottle doesn't have these issues. - -Throttling algorithm -==================== - -AutoThrottle algorithm adjusts download delays based on the following rules: - -1. spiders always start with a download delay of - :setting:`AUTOTHROTTLE_START_DELAY`; -2. when a response is received, the target download delay is calculated as - ``latency / N`` where ``latency`` is a latency of the response, - and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`. -3. download delay for next requests is set to the average of previous - download delay and the target download delay; -4. latencies of non-200 responses are not allowed to decrease the delay; -5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater - than :setting:`AUTOTHROTTLE_MAX_DELAY` - -.. note:: The AutoThrottle extension honours the standard Scrapy settings for - concurrency and delay. This means that it will respect - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and - never set a download delay lower than :setting:`DOWNLOAD_DELAY`. - -.. _download-latency: - -In Scrapy, the download latency is measured as the time elapsed between -establishing the TCP connection and receiving the HTTP headers. - -Note that these latencies are very hard to measure accurately in a cooperative -multitasking environment because Scrapy may be busy processing a spider -callback, for example, and unable to attend downloads. However, these latencies -should still give a reasonable estimate of how busy Scrapy (and ultimately, the -server) is, and this extension builds on that premise. - -.. reqmeta:: autothrottle_dont_adjust_delay - -Prevent specific requests from triggering slot delay adjustments -================================================================ - -AutoThrottle adjusts the delay of download slots based on the latencies of -responses that belong to that download slot. The only exceptions are non-200 -responses, which are only taken into account to increase that delay, but -ignored if they would decrease that delay. - -You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to -``True`` in any request to prevent its response latency from impacting the -delay of its download slot: - -.. code-block:: python - - from scrapy import Request - - Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) - -Note, however, that AutoThrottle still determines the starting delay of every -download slot by setting the ``download_delay`` attribute on the running -spider. If you want AutoThrottle not to impact a download slot at all, in -addition to setting this meta key in all requests that use that download slot, -you might want to set a custom value for the ``delay`` attribute of that -download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. - -Settings -======== - -The settings used to control the AutoThrottle extension are: - -* :setting:`AUTOTHROTTLE_ENABLED` -* :setting:`AUTOTHROTTLE_START_DELAY` -* :setting:`AUTOTHROTTLE_MAX_DELAY` -* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` -* :setting:`AUTOTHROTTLE_DEBUG` -* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -* :setting:`CONCURRENT_REQUESTS_PER_IP` -* :setting:`DOWNLOAD_DELAY` - -For more information see :ref:`autothrottle-algorithm`. - -.. setting:: AUTOTHROTTLE_ENABLED - -AUTOTHROTTLE_ENABLED -~~~~~~~~~~~~~~~~~~~~ - -Default: ``False`` - -Enables the AutoThrottle extension. - -.. setting:: AUTOTHROTTLE_START_DELAY - -AUTOTHROTTLE_START_DELAY -~~~~~~~~~~~~~~~~~~~~~~~~ - -Default: ``5.0`` - -The initial download delay (in seconds). - -.. setting:: AUTOTHROTTLE_MAX_DELAY - -AUTOTHROTTLE_MAX_DELAY -~~~~~~~~~~~~~~~~~~~~~~ - -Default: ``60.0`` - -The maximum download delay (in seconds) to be set in case of high latencies. - -.. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY - -AUTOTHROTTLE_TARGET_CONCURRENCY -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Default: ``1.0`` - -Average number of requests Scrapy should be sending in parallel to remote -websites. It must be higher than ``0.0``. - -By default, AutoThrottle adjusts the delay to send a single -concurrent request to each of the remote websites. Set this option to -a higher value (e.g. ``2.0``) to increase the throughput and the load on remote -servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value -(e.g. ``0.5``) makes the crawler more conservative and polite. - -Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected -when AutoThrottle extension is enabled. This means that if -``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number -of concurrent requests. - -At every given time point Scrapy can be sending more or less concurrent -requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested -value the crawler tries to approach, not a hard limit. - -.. setting:: AUTOTHROTTLE_DEBUG - -AUTOTHROTTLE_DEBUG -~~~~~~~~~~~~~~~~~~ - -Default: ``False`` - -Enable AutoThrottle debug mode which will display stats on every response -received, so you can see how the throttling parameters are being adjusted in -real time. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 122022bdb..78a09f2c3 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -629,7 +629,6 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: * :reqmeta:`allow_offsite` -* :reqmeta:`autothrottle_dont_adjust_delay` * :reqmeta:`bindaddress` * :reqmeta:`cookiejar` * :reqmeta:`dont_cache` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3e8de9107..6ea336458 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -502,6 +502,17 @@ Default: ``100`` Maximum number of concurrent items (per response) to process in parallel in :ref:`item pipelines `. +.. setting:: CONCURRENT_REQUESTS + +CONCURRENT_REQUESTS +------------------- + +Default: ``16`` + +Maximum number of total concurrent requests allowed. + +.. seealso:: :ref:`throttling` + .. setting:: DEFAULT_DROPITEM_LOG_LEVEL DEFAULT_DROPITEM_LOG_LEVEL @@ -1128,7 +1139,6 @@ Default: "scrapy.extensions.feedexport.FeedExporter": 0, "scrapy.extensions.logstats.LogStats": 0, "scrapy.extensions.spiderstate.SpiderState": 0, - "scrapy.extensions.throttle.AutoThrottle": 0, } A dict containing the extensions available by default in Scrapy, and their @@ -1509,26 +1519,6 @@ Example:: NEWSPIDER_MODULE = 'mybot.spiders_dev' -.. setting:: RANDOMIZE_DOWNLOAD_DELAY - -RANDOMIZE_DOWNLOAD_DELAY ------------------------- - -Default: ``True`` - -If enabled, Scrapy will wait a random amount of time (between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`) while fetching requests from the same -website. - -This randomization decreases the chance of the crawler being detected (and -subsequently blocked) by sites which analyze requests looking for statistically -significant similarities in the time between their requests. - -The randomization policy is the same used by `wget`_ ``--random-wait`` option. - -If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. - -.. _wget: https://www.gnu.org/software/wget/manual/wget.html - .. setting:: REACTOR_THREADPOOL_MAXSIZE REACTOR_THREADPOOL_MAXSIZE diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index ce4d5ee00..b3bc7e3de 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -4,89 +4,101 @@ Throttling ========== -Sending too many requests too quickly can `overload websites`_. To avoid that, -you must throttle_ your requests. Scrapy can :ref:`throttle requests -`, :ref:`handle backoff `, and much more. +Sending too many requests too quickly can `overwhelm websites`_. +:ref:`Throttling ` and :ref:`backoff ` aim to +prevent that. -.. _overload websites: https://en.wikipedia.org/wiki/Denial-of-service_attack -.. _throttle: https://en.wikipedia.org/wiki/Bandwidth_throttling +.. _overwhelm websites: https://en.wikipedia.org/wiki/Denial-of-service_attack .. _basic-throttling: Concurrency and delay ===================== -Use the following :ref:`settings ` to configure the default -throttling for each :ref:`throttling scope `: +Requests are throttled on a **per-domain basis** by default [1]_. This allows +efficient crawling of multiple sites simultaneously. -- .. setting:: THROTTLING_CONCURRENCY +Each domain and subdomain is treated separately: requests to +``books.toscrape.com`` and ``quotes.toscrape.com`` each have their own +throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. - **THROTTLING_CONCURRENCY** (default: ``1``) +The main throttling :ref:`settings ` are: - The maximum number of concurrent requests. +- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN -- .. setting:: THROTTLING_DELAY + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1``) - **THROTTLING_DELAY** (default: ``1.0``) + Maximum number of simultaneous requests per domain. - The minimum seconds to wait between consecutive requests. + It defines a number of “slots” per domain. Each slot can send 1 request at + a time: it sends a request, waits for the response, then sends the next + request, and so on. -- .. setting:: THROTTLING_JITTER +- .. setting:: DOWNLOAD_DELAY - **THROTTLING_JITTER** (default: ``0.5``, i.e. ±50%) + :setting:`DOWNLOAD_DELAY` (default: ``1.0``) - Randomize delays by this factor, i.e. the final delay is a random value - between ``delay*(1-jitter)`` and ``delay*(1+jitter)``. + Minimum seconds between any two requests to the same domain. + + Even if you have multiple slots, requests to the same domain cannot be sent + more frequently than this delay. + +- .. setting:: DOWNLOAD_DELAY_PER_SLOT + + :setting:`DOWNLOAD_DELAY_PER_SLOT` (default: ``1.0``) + + Minimum seconds between requests in the same slot. + + If a slot sends a request and receives its response before this delay has + elapsed, it must wait before sending the next request. The wait time is + measured from when the previous request was sent. + +For example, with ``CONCURRENT_REQUESTS_PER_DOMAIN = 2``, ``DOWNLOAD_DELAY = 0.3``, +and ``DOWNLOAD_DELAY_PER_SLOT = 1.0``, sending 3 requests to the same domain +would result in: + +.. code-block:: text + + T=0.0s: Request 1 sent (slot 1) + T=0.3s: Request 2 sent (slot 2, respects same-domain delay) + T=0.6s: Request 3 must wait (same-domain delay satisfied, but slot 1 needs 1.0s) + T=1.0s: Request 3 sent (slot 1 can now be reused) + +When configuring these settings, note that: + +- :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``. + +- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``. + This happens because all slots must wait for the delay between requests, + preventing them from sending requests simultaneously. + +.. [1] You can :ref:`customize ` how requests are grouped + for throttling, but domain-based throttling works well in most cases. For + more complex domain grouping strategies, see + :ref:`alternative-domain-throttling`. - It can be set to a 2-item list with low and high factors, e.g. - ``[-0.1, 0.3]`` to randomize delays between ``delay*0.9`` and - ``delay*1.3``. .. setting:: THROTTLING_SCOPES +.. _per-domain-throttling: -Use **THROTTLING_SCOPES** (default: ``{}``) to override these values for -specific throttling scopes: +Per-domain throttling +===================== - .. code-block:: python +The :setting:`THROTTLING_SCOPES` setting allows you to customize throttling behavior +for specific domains [1]_. - THROTTLING_SCOPES = { - "books.toscrape.com": {"concurrency": 16, "delay": 0.0}, - "example.com": {"jitter": 0.2}, - } +Its default value allows faster crawling of the testing website using during +the :ref:`tutorial ` while maintaining conservative defaults +for other domains: -:setting:`THROTTLING_SCOPES` can also :ref:`override backoff settings -`. +.. code-block:: python -When setting these values, note that: + THROTTLING_SCOPES = { + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.0}, + } -- :setting:`CONCURRENT_REQUESTS` effectively caps concurrency for any - throttling scope. - -- When higher than response time, delay effectively limits concurrency to - ``1``. - - -.. _crawl-delay: - -Crawl-Delay directive ---------------------- - -`Crawl-Delay `__ -is a non-standard ``robots.txt`` directive that indicates a number of seconds to -wait between requests. - -.. setting:: THROTTLING_ROBOTSTXT_OBEY -.. setting:: THROTTLING_ROBOTSTXT_MAX_DELAY - -If :setting:`ROBOTSTXT_OBEY` and **THROTTLING_ROBOTSTXT_OBEY** are -``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`THROTTLING_CONCURRENCY` and :setting:`THROTTLING_DELAY`. Concurrency -is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at -**THROTTLING_ROBOTSTXT_MAX_DELAY** (default: ``60.0``). - -If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it -will be respected, but a warning will be logged about the discrepancy with -``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. +Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are +covered later on. .. _backoff: @@ -94,136 +106,148 @@ will be respected, but a warning will be logged about the discrepancy with Backoff ======= -When a response or network error warrants backoff, `exponential backoff`_ is -used to reduce request rate. +When servers respond with rate limiting errors (like HTTP 429) or network +timeouts occur, request rate is automatically reduced using `exponential +backoff`_. .. _exponential backoff: https://en.wikipedia.org/wiki/Exponential_backoff -In such cases, every new request with the same throttling scope is sent with -its delay multiplied by some factor (up to some maximum) or set to some minimum -value (if it was lower), until a request gets a response that does not require -backoff. Once a response that does not require backoff is received, the delay -is gradually reduced back to its original value. - -The following settings control backoff behavior: +The key settings are: - .. setting:: BACKOFF_HTTP_CODES - **BACKOFF_HTTP_CODES** (default: ``[429, 502, 503, 504, 520, 521, 522, 523, 524]``) + :setting:`BACKOFF_HTTP_CODES` (default: ``[429, 502, 503, 504, 520, 521, 522, 523, 524]``) - HTTP response status codes that warrant backoff. + HTTP status codes that trigger backoff. - Usually, all codes here should be in :setting:`RETRY_HTTP_CODES` as well, - but not all codes in :setting:`RETRY_HTTP_CODES` need to be here: some bad - responses may require a retry without backoff. +- .. setting:: BACKOFF_DELAY_FACTOR -- .. setting:: BACKOFF_EXCEPTIONS + :setting:`BACKOFF_DELAY_FACTOR` (default: ``2.0``) - **BACKOFF_EXCEPTIONS** - - Default: - - .. code-block:: python - - [ - "twisted.internet.defer.TimeoutError", - "twisted.internet.error.TimeoutError", - "twisted.internet.error.TCPTimedOutError", - "twisted.web.client.ResponseFailed", - ] - - Exception classes that warrant backoff. Strings are interpreted as import - paths. - - Usually, all exceptions here should be in :setting:`RETRY_EXCEPTIONS` as - well, but not all exceptions in :setting:`RETRY_EXCEPTIONS` need to be - here: some errors may require a retry without backoff. - -- .. setting:: BACKOFF_FACTOR - - **BACKOFF_FACTOR** (default: ``2.0``) - - The factor by which the delay is multiplied for each new request sent to a - given throttling scope during backoff. + Each backoff multiplies delay by this factor (2x, 4x, 8x, etc.). - .. setting:: BACKOFF_MAX_DELAY - **BACKOFF_MAX_DELAY** (default: ``300.0``) + :setting:`BACKOFF_MAX_DELAY` (default: ``300.0``) - The maximum delay that can be applied during backoff. If the delay exceeds - this value, it will be capped at this value. + Maximum delay cap to prevent excessively long waits. -- .. setting:: BACKOFF_MIN_DELAY - **BACKOFF_MIN_DELAY** (default: ``1.0``) +.. _rampup: - The minimum delay that can be applied during backoff. If the delay is less - than this value, it will be set to this value. Must be higher than ``0.0``. +Rampup +====== -- .. setting:: BACKOFF_JITTER - - **BACKOFF_JITTER** (default: ``0.1``) - - Overrides :setting:`THROTTLING_JITTER` during backoff. - -When a throttling scope is configured with a **concurrency higher than 1**, -backoff is handled separately per concurrency slot. If at some point all -concurrency slots reach the maximum backoff delay, a “concurrency backoff” -starts, controlled by the following setting: - -- .. setting:: BACKOFF_CONCURRENCY_DECREASE_FACTOR - - **BACKOFF_CONCURRENCY_DECREASE_FACTOR** (default: ``0.5``) - - The factor by which the concurrency is decreased during concurrency - backoff. - -.. _scope-backoff: - -Backoff settings can be overridden per throttling scope using -:setting:`THROTTLING_SCOPES`: +When using APIs that charge per request, like web scraping APIs, you often want +to maximize throughput while staying within rate limits. To do that, set +``"rampup"`` to ``True`` in :setting:`THROTTLING_SCOPES`: .. code-block:: python + :caption: ``settings.py`` - { - "example.com": { - "backoff": { - "http_codes": [429, 503], - "exceptions": ["builtins.IOError"], - "factor": 1.2, - "max_delay": 180.0, - "min_delay": 5.0, - "jitter": [0.01, 0.33], - "concurrency_decrease_factor": 0.8, - } + THROTTLING_SCOPES = { + "api.toscrape.com": { + "rampup": True, }, } +Rampup increases concurrency or lowers delay as needed based on the following +setting: + +- .. setting:: RAMPUP_BACKOFF_TARGET + + :setting:`RAMPUP_BACKOFF_TARGET` (default: ``1``) + + Target number of backoff responses per rampup window, indicating optimal + throughput. Can be a range like ``[1, 3]``. + .. _retry-after: +.. _rate-limiting-headers: Rate limiting headers ---------------------- +===================== -The `Retry-After +Servers may include `Retry-After `__ -and the `RateLimit-Reset +or `RateLimit-Reset `__ -HTTP response headers indicate how long to wait before making a follow-up -request. - -They are taken into account during :ref:`backoff `: their value is -read (the highest if both headers are present), capped at -:setting:`BACKOFF_MAX_DELAY`, and used as a minimum delay, i.e. it is used if -higher than the current delay but ignored if lower. +headers to indicate when you should make your next request. These headers are +respected automatically during :ref:`backoff `, using their values as +minimum delays (capped at :setting:`BACKOFF_MAX_DELAY`). .. seealso:: :setting:`REDIRECT_MAX_DELAY` +.. _crawl-delay: + +robots.txt +========== + +`Crawl-Delay `__ +is a non-standard ``robots.txt`` directive that indicates a number of seconds +to wait between requests. + +.. setting:: THROTTLING_ROBOTSTXT_OBEY +.. setting:: THROTTLING_ROBOTSTXT_MAX_DELAY + +If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are +``True`` (default), valid ``Crawl-Delay`` directives override +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. Concurrency +is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at +:setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). + +If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it +will be respected, but a warning will be logged about the discrepancy with +``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. + + +.. _per-request-throttling: + +Per-request throttling +====================== + +Sometimes you need different throttling behavior for individual requests or for +request groups that are not tied to a specific domain. + +For example, you might want to throttle API endpoints differently than web +pages on the same domain, group requests by content type (images vs HTML), or +apply different throttling based on request priority. + +.. reqmeta:: throttling_scopes + +Use the ``throttling_scopes`` request metadata to assign requests to custom +throttling groups: + +.. code-block:: python + + Request("https://api.example/", meta={"throttling_scopes": "api"}) + +You can also assign multiple throttling groups to a single request: + +.. code-block:: python + + Request("https://api.example/users", meta={"throttling_scopes": {"api", "users"}}) + +You can then use the :setting:`THROTTLING_SCOPES` setting to customize +throttling for such requests: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPES = { + "api": {"concurrency": 2}, + "users": {"delay": 5.0}, + } + +.. note:: These custom throttling groups persist through redirects. For + redirect-aware throttling assignment, see :ref:`custom-throttling-scopes`. + + .. _throttling-scopes: -Scopes -====== +Throttling scopes +================= Throttling scopes represent aspects of requests that can be throttled independently. @@ -232,27 +256,6 @@ independently. For future reference, the “throttling scope” name was taken from https://www.ietf.org/archive/id/draft-polli-ratelimit-headers-02.html#section-1.4-4.4 -.. _default-throttling-scopes: - -Default throttling scopes --------------------------- - -By default, each request has a single throttling scope representing the domain -or subdomain of the target URL. That is, when you set the concurrency or delay -of a throttling scope, it applies to all requests made to that domain or -subdomain. - -For example, https://books.toscrape.com and -https://books.toscrape.com/catalogue/page-2.html both get a -``books.toscrape.com`` throttling scope. - -Note however that subdomains are treated as separate throttling scopes by -default. For example, https://toscrape.com gets a ``toscrape.com`` throttling -scope, and ``books.toscrape.com`` and ``toscrape.com`` are considered -unrelated throttling scopes. If you want to change this behavior, see -:ref:`alternative-domain-throttling`. - - .. _custom-throttling-scopes: Customizing throttling scopes @@ -260,24 +263,9 @@ Customizing throttling scopes There are 2 ways to customize throttling scopes. -.. reqmeta:: throttling_scopes - -For simple use cases, you can use the ``throttling_scopes`` request metadata -key: - -.. code-block:: python - - Request("https://example.com/", meta={"throttling_scopes": "foo"}) - Request("https://example.com/", meta={"throttling_scopes": {"foo", "bar"}}) - Request("https://example.com/", meta={"throttling_scopes": {"foo": 1.0, "bar": 2.5}}) - -.. note:: Throttling scopes set through request metadata remain through the - request lifetime, e.g. throught redirects, even if those change the request - URL. - .. setting:: THROTTLING_MANAGER -For anything else, set **THROTTLING_MANAGER** (default: +For anything else, set :setting:`THROTTLING_MANAGER` (default: :class:`~scrapy.throttling.ThrottlingManager`) to a :ref:`component ` that implements the :class:`~scrapy.throttling.ThrottlingManagerProtocol` protocol (or its import @@ -289,41 +277,92 @@ path as a string): THROTTLING_MANAGER = "myproject.throttling.MyThrottlingManager" +.. _multi-throttling-scopes: + +Handling of multiple throttling scopes +-------------------------------------- + +When a request has multiple throttling scopes, it is not sent until all of its +throttling scopes allow it. + + .. _throttling-quotas: -Quotas -====== +Throttling quotas +----------------- When different requests can consume different amounts of a throttling scope, -you can express this using quotas. +you can express this using **throttling quotas**. -In the :reqmeta:`throttling_scopes` request metadata key and in the -:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scopes` method you use -a :class:`dict` structure where keys are throttling scopes and values are -:class:`float` that indicate the amount of a scope that the request is expected -to consume (it does not need to be exact). +.. setting:: THROTTLING_WINDOW -By default, those values are ignored. However, if a call to -:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_response_throttling` or -:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_exception_throttling` -reports available quotas for one or more throttling scopes, request quotas will -start being tracked and determine which requests can be sent and which cannot. +Use the :setting:`THROTTLING_WINDOW` setting (default: ``60.0``) or the ``"window"`` +key in the :setting:`THROTTLING_SCOPES` setting to define the time window after +which throttling quotas are reset. -In fact, everything else being equal, -:class:`~scrapy.pqueues.ScrapyPriorityQueue` prioritizes requests that consume -a higher portion of the available quota, to minimize the risk of those -requests getting stuck. +Then use the :setting:`THROTTLING_SCOPES` setting to define the throttling +quotas for each throttling scope: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPES = { + "api.toscrape.com": { + "quota": 500.0, + }, + } + +Then, in the :reqmeta:`throttling_scopes` request metadata key or in the return +value of the :meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scopes` +method, define a :class:`dict` where keys are throttling scopes and values are +:class:`float` values that indicate the expected quota consumption (it does not +need to be exact). + +Everything else being equal, :class:`~scrapy.pqueues.ScrapyPriorityQueue` will +prioritize requests that consume a higher portion of the available throttling +quota, to minimize the risk of those requests getting stuck. -API -=== +.. _custom-throttling-scope-managers: -.. autoclass:: scrapy.throttling.ThrottlingManagerProtocol - :members: - :member-order: bysource +Customizing throttling scope managers +------------------------------------- -.. autoclass:: scrapy.throttling.ThrottlingManager +.. setting:: THROTTLING_SCOPE_MANAGER +The :setting:`THROTTLING_SCOPE_MANAGER` setting (default: +:class:`~scrapy.throttling.ThrottlingScopeManager`) is a :ref:`component +` that implements the +:class:`~scrapy.throttling.ThrottlingScopeManagerProtocol` (or its import path +as a string): + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPE_MANAGER = "myproject.throttling.MyThrottlingScopeManager" + +For each throttling scope, an instance of this class is created to manage any +gradual :ref:`backoff ` or :ref:`rampup ` required at run +time. + +You can implement your own throttling scope manager if you wish to change the +backoff or rampup behavior beyond what settings allow. + +You can also define a custom throttling scope manager for a specific throttling +scope by setting the ``"manager"`` key in the :setting:`THROTTLING_SCOPES` +setting: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPES = { + "api.toscrape.com": { + "manager": "myproject.throttling.MyThrottlingScopeManager", + }, + } + + +.. _throttling-examples: Examples ======== @@ -334,8 +373,7 @@ Alternative domain throttling ----------------------------- If you are not happy with the :ref:`default throttling scope behavior -` with regards to domains and subdomains, you can -change it. +` with regards to domains and subdomains, you can change it. Alternative approaches include: @@ -415,505 +453,327 @@ Alternative approaches include: at the same time, because that would sum 40 concurrency, and ``toscrape.com`` requests are limited to 32. +.. _endpoints-throttling: +Endpoint-specific throttling +---------------------------- +To apply different throttling settings to different endpoints of the same +domain and not enforce any common throttling, effectively treating them as +different domains: - - - - - - - -Requests can be assigned 1 or more throttling scopes, each with a value. - - - -This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY` -setting, which is enabled by default. - -When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced -per IP address instead of per domain. - -Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain -concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response -time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective -concurrency for that domain is 1. When testing throttling configurations, it -usually makes sense to lower :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` first, -and only increase :setting:`DOWNLOAD_DELAY` once -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is 1 but a higher throttling is -desired. - -.. _spider-download_delay-attribute: - -.. note:: - - This delay can be set per spider using :attr:`download_delay` spider attribute. - -It is also possible to change this setting per domain, although it requires -non-trivial code. See the implementation of the :ref:`AutoThrottle -` extension for an example. - -.. - TODO: Add a section about the handling of subdomains. By default, Scrapy - should treat subdomains as separate slots, but it should be easy to change - that behavior for specific domains, maybe with some new setting. - -Throttling scopes -================== - -.. - TODO: Cover what they are, what their default values are, how to change - them easily on a per-request base. - - -Settings -======== - - -.. setting:: DOWNLOAD_SLOTS - -DOWNLOAD_SLOTS --------------- - -Default: ``{}`` - -Allows to define concurrency/delay parameters on per slot (domain) basis: +- Implement a :ref:`throttling manager ` that sets + endpoint-specific throttling scopes for that domain: .. code-block:: python - DOWNLOAD_SLOTS = { - "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False}, - "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + from scrapy.throttling import ThrottlingManager, scope_cache + from scrapy.utils.httpobj import urlparse_cached + + + class MyThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + parsed_url = urlparse_cached(request) + if parsed_url.netloc != "api.toscrape.com": + return await super().get_scopes(request) + return f"{parsed_url.netloc}{parsed_url.path}" + +- Use the :setting:`THROTTLING_SCOPES` setting to set different throttling + settings per endpoint: + + .. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPES = { + "api.toscrape.com/fast-endpoint": {"concurrency": 1000, "delay": 0.08}, + "api.toscrape.com/slow-endpoint": {"delay": 5.0}, } -.. note:: - For other downloader slots default settings values will be used: +.. _web-scraping-api-throttling: - - :setting:`DOWNLOAD_DELAY`: ``delay`` - - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency`` - - :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay`` - - - -Global Request Limits -====================== - -The simplest form of throttling is limiting the total number of concurrent requests across your entire spider. - -CONCURRENT_REQUESTS -------------------- - -The :setting:`CONCURRENT_REQUESTS` setting controls the maximum number of requests that can be processed simultaneously across all domains: - -.. code-block:: python - - # settings.py - CONCURRENT_REQUESTS = 16 # Default value - -This is a global limit that affects all requests regardless of their target domain. Setting it to a lower value will make your spider more conservative and polite, but slower. Setting it higher will make requests faster but may overwhelm servers or your own network connection. - -**When to use**: This is your first line of defense against sending too many requests at once. Good default values are typically between 8-32 depending on your needs and the target servers' capacity. - -Per-Domain Request Limits -========================== - -More sophisticated throttling involves limiting requests on a per-domain basis, which is usually more appropriate since different servers have different capacities. - -CONCURRENT_REQUESTS_PER_DOMAIN -------------------------------- - -The :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting limits concurrent requests to each individual domain: - -.. code-block:: python - - # settings.py - CONCURRENT_REQUESTS_PER_DOMAIN = 8 # Default value - -This means that even if you have :setting:`CONCURRENT_REQUESTS` set to 32, no single domain will receive more than 8 concurrent requests. This prevents one fast-responding domain from monopolizing all your concurrent request slots. - -**Example**: If you're scraping both ``example.com`` and ``other-site.com``, each domain will be limited to 8 concurrent requests, allowing a total of up to 16 concurrent requests (but still subject to the global :setting:`CONCURRENT_REQUESTS` limit). - -CONCURRENT_REQUESTS_PER_IP +Web scraping API throttling --------------------------- -Similar to per-domain limits, :setting:`CONCURRENT_REQUESTS_PER_IP` limits concurrent requests per IP address: +Imagine you are sending requests to a web scraping API, e.g. to avoid bans. +Unless that API provides a Scrapy plugin to make it easier to use, you may want +to: -.. code-block:: python +- Use the :setting:`THROTTLING_SCOPES` setting to increase concurrency for + API requests. For example: - # settings.py - CONCURRENT_REQUESTS_PER_IP = 1 # Default value + .. code-block:: python + :caption: ``settings.py`` -This is useful when multiple domains resolve to the same IP address (common with CDNs or shared hosting). The IP-based limit takes precedence over the domain-based limit when they conflict. + THROTTLING_SCOPES = { + "api.toscrape.com": {"concurrency": 1000, "delay": 0.08}, + } -DOWNLOAD_DELAY --------------- +- Implement a :ref:`throttling manager ` that: -While concurrency limits control how many requests are sent simultaneously, :setting:`DOWNLOAD_DELAY` controls the time delay between requests to the same domain: + - Adds a throttling scope for the URL being scraped. -.. code-block:: python + For example, if you request + ``https://api.toscrape.com/?url=https://example.com``, by default it + will get a ``api.toscrape.com`` throttling scope, but it should also + get the ``example.com`` throttling scope: - # settings.py - DOWNLOAD_DELAY = 3 # Wait 3 seconds between requests to the same domain + .. code-block:: python -This setting introduces a delay between consecutive requests to the same domain/slot. It's applied per-domain, so requests to different domains are not affected by each other's delays. + from urllib.parse import urlparse -The delay can be randomized using :setting:`RANDOMIZE_DOWNLOAD_DELAY`: + from scrapy.throttling import add_scope, ThrottlingManager, scope_cache + from scrapy.utils.httpobj import urlparse_cached + from w3lib.url import url_query_parameter -.. code-block:: python - # settings.py - DOWNLOAD_DELAY = 3 - RANDOMIZE_DOWNLOAD_DELAY = True # Default - # This will use delays between 1.5 and 4.5 seconds (0.5 * DOWNLOAD_DELAY to 1.5 * DOWNLOAD_DELAY) + class MyThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + if urlparse_cached(request).netloc != "api.toscrape.com": + return scopes + target_url = url_query_parameter(request.url, "url") + if not target_url: + return scopes + target_domain = urlparse(target_url).netloc + return add_scope(scopes, target_domain) -**When to use**: Download delays are particularly useful for websites that are sensitive to request frequency but can handle multiple concurrent connections. They're also helpful for mimicking human-like browsing patterns. + - Can differentiate between exhaustion of the target website and + exhaustion of the API itself. For example: -**Note**: A global download delay doesn't make sense because it would unnecessarily slow down requests to different domains. The per-domain approach allows you to be respectful to each server individually while maintaining overall efficiency. + .. code-block:: python -Custom Download Slots -====================== + from scrapy.throttling import ThrottlingManager + from scrapy.utils.httpobj import urlparse_cached -For more advanced scenarios, you can customize how requests are grouped for throttling purposes using download slots. -Understanding Download Slots ------------------------------ + class MyThrottlingManager(ThrottlingManager): + async def get_response_backoff(self, response): + if ( + urlparse_cached(response.request).netloc != "api.toscrape.com" + or response.status != 200 + ): + return await super().get_response_backoff(response) + upstream_status_code = int( + response.headers.get("X-Upstream-Status-Code", b"200") + ) + upstream_response = response.__class__( + response.url, + status=upstream_status_code, + headers=response.headers, + body=response.body, + ) + return await super().get_response_backoff(upstream_response) -By default, Scrapy groups requests by their domain name for throttling purposes. Each domain gets its own "download slot" with its own concurrency limits and delays. However, you can customize this grouping: -.. code-block:: python +.. _cost-smoothing-throttling: - # In your spider - def start_requests(self): - # Default: requests to example.com go to "example.com" slot - yield scrapy.Request("https://example.com/page1") - yield scrapy.Request("https://example.com/page2") - - # Custom: group these requests differently - yield scrapy.Request( - "https://api.example.com/fast", meta={"download_slot": "api_fast"} - ) - yield scrapy.Request( - "https://api.example.com/slow", meta={"download_slot": "api_slow"} - ) - -Per-Slot Configuration +Cost-capped throttling ---------------------- -You can configure different throttling settings for different slots using :setting:`DOWNLOAD_SLOTS`: +Imagine you are using an API that charges different requests differently, e.g. +based on the features used, and you want to limit how much you spend per time +window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas +` for that: -.. code-block:: python +- Implement a :ref:`throttling manager ` that: - # settings.py - DOWNLOAD_SLOTS = { - "api_fast": { - "concurrency": 1, - "delay": 0.5, - }, - "api_slow": { - "concurrency": 1, - "delay": 5.0, - }, - "images": { - "concurrency": 8, - "delay": 0, - }, - } + - Sets a ``cost`` throttling scope on each request to some estimation based + e.g. on request URL parameters: -**Use cases**: -- Different API endpoints with different rate limits -- Separating expensive operations (like browser rendering) from cheap ones -- Treating different subdomains with different politeness levels -- Grouping requests by authentication context + .. code-block:: python -Advanced Throttling System -=========================== + from scrapy.utils.httpobj import urlparse_cached + from scrapy.throttling import ThrottlingManager, scope_cache -For complex scraping scenarios that require fine-grained control over throttling, Scrapy provides an advanced throttling scope system that goes beyond simple per-domain limits. -Throttling Buckets ------------------- + class MyThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + parsed_url = urlparse_cached(request) + if parsed_url.netloc != "api.toscrape.com": + return scopes + return add_scope(scopes, "cost", estimate_request_cost(request)) -The new throttling system introduces the concept of "throttling scopes" - resources that requests consume and that can become temporarily unavailable when limits are exceeded. Unlike download slots, a single request can require multiple scopes, enabling multi-dimensional throttling. + - Reports the actual cost during response parsing: -Enabling Throttling Buckets ----------------------------- + .. code-block:: python -.. code-block:: python + from scrapy.throttling import ThrottlingManager - # settings.py - THROTTLING_ENABLED = True - THROTTLING_MANAGER = "scrapy.throttling.DefaultBucketManager" -Basic Bucket Usage ------------------- + class MyThrottlingManager(ThrottlingManager): + async def get_response_backoff(self, response): + scopes = await super().get_response_backoff(response) + if "cost" not in scopes: + return scopes + actual_cost = float(response.headers.get("X-Actual-Cost", b"0")) + return update_scope_backoff(scopes, "cost", consumed_quota=actual_cost) -The simplest scope configuration replicates domain-based throttling: +- Use the :setting:`THROTTLING_SCOPES` setting to set a maximum cost per time + window: -.. code-block:: python + .. code-block:: python + :caption: ``settings.py`` - # Custom scope manager - class MyBucketManager: - def get_request_scopes(self, request, spider): - domain = urlparse(request.url).netloc - return {domain: 1.0} # Consume 1 unit of the domain scope + THROTTLING_SCOPES = { + "cost": {"quota": 100.0}, + } - def process_response(self, response, request, spider): - if response.status == 429: # Too Many Requests - domain = urlparse(request.url).netloc - # Throttle this domain for 60 seconds - self.throttle_scope(domain, delay=60) + This will allow you to spend up to 100.0 units of cost per time window + (default: 60 seconds) before throttling kicks in. -Multi-Dimensional Throttling ------------------------------ -The real power comes from using multiple scopes per request: +.. _throttling-settings: -.. code-block:: python +Additional settings +=================== +- .. setting:: BACKOFF_EXCEPTIONS - def get_request_scopes(self, request, spider): - scopes = {} + :setting:`BACKOFF_EXCEPTIONS` - # Domain-based throttling - domain = urlparse(request.url).netloc - scopes[domain] = 1.0 + Default: - # API feature-based throttling - if "browser=true" in request.url: - scopes["browser_rendering"] = 1.0 + .. code-block:: python - if "extract=true" in request.url: - scopes["ai_extraction"] = 5.0 # More expensive + [ + "twisted.internet.defer.TimeoutError", + "twisted.internet.error.TimeoutError", + "twisted.internet.error.TCPTimedOutError", + "twisted.web.client.ResponseFailed", + ] - # Geographic throttling - if "region=eu" in request.url: - scopes["eu_datacenter"] = 1.0 + Exception classes that trigger backoff. Strings are interpreted as import + paths. - return scopes + .. seealso:: :setting:`RETRY_EXCEPTIONS` -Cost-Based Throttling ---------------------- +- .. setting:: BACKOFF_JITTER -Some APIs charge different amounts for different operations. The scope system supports fractional consumption: + :setting:`BACKOFF_JITTER` (default: ``0.1``) -.. code-block:: python + Overrides :setting:`RANDOMIZE_DOWNLOAD_DELAY` during backoff. - def get_request_scopes(self, request, spider): - scopes = {"api_credits": 1.0} # Default cost +- .. setting:: BACKOFF_MIN_DELAY - if "operation=expensive" in request.url: - scopes["api_credits"] = 10.0 # Costs 10x more + :setting:`BACKOFF_MIN_DELAY` (default: ``1.0``) - return scopes + Minimum delay during :ref:`backoff `. +- .. setting:: BACKOFF_WINDOW - def process_response(self, response, request, spider): - # Update actual consumption based on response - if "X-Actual-Cost" in response.headers: - actual_cost = float(response.headers["X-Actual-Cost"]) - # Could update scope consumption here for better accuracy + :setting:`BACKOFF_WINDOW` (default: ``60.0``) -Responding to Server Signals ------------------------------ + During :ref:`backoff `, after a non-backoff response is received, + do not take the next step in backoff reduction until this amount of time + has passed and no new backoff feedback has been received. -The scope system can respond intelligently to server throttling signals: + The number of seconds that need to pass since the last non-backoff response + without any other for the backoff to move towards the original throttling + configuration. -.. code-block:: python +- .. setting:: DELAYED_REQUESTS_WARN_THRESHOLD - def process_response(self, response, request, spider): - if response.status == 429: - # Check for Retry-After header - retry_after = response.headers.get("Retry-After") - if retry_after: - delay = int(retry_after) - else: - delay = 60 # Default backoff + :setting:`DELAYED_REQUESTS_WARN_THRESHOLD` (default: ``500``) - # Determine which scope to throttle based on response - if "rate limit exceeded for API key" in response.text: - self.throttle_scope("api_key_global", delay=delay) - elif "too many requests to this endpoint" in response.text: - endpoint = self._extract_endpoint(request.url) - self.throttle_scope(f"endpoint_{endpoint}", delay=delay) + While throttled, requests in the :ref:`scheduler ` remain + in the scheduler. - elif response.status == 503: - # Service unavailable - throttle the entire domain - domain = urlparse(request.url).netloc - self.throttle_scope(domain, delay=300) # 5 minutes + However, requests sent with :meth:`engine.download() + ` bypass the scheduler. This + includes requests sent by some built-in :ref:`components + ` and :ref:`inline requests `. -Configuration -------------- + When such requests are throttled, they are paused and kept in memory, along + with any run time context from the code that is sending them. If they + accumulate, they can become a memory issue that may require you to rethink + your throttling parameters or crawl strategy. -The throttling system provides several configuration options: + :setting:`DELAYED_REQUESTS_WARN_THRESHOLD` defines a threshold for such + requests. The first time that this many such requests are being throttled + at the same time, a warning is issued. -.. code-block:: python +- .. setting:: RANDOMIZE_DOWNLOAD_DELAY - # settings.py - THROTTLING_ENABLED = True - THROTTLING_MANAGER = "myproject.throttling.CustomBucketManager" + :setting:`RANDOMIZE_DOWNLOAD_DELAY` (default: ``True``) - # Maximum number of delayed requests to keep in memory - THROTTLING_MAX_DELAYED_REQUESTS = 1000 + Randomize delays by this factor, e.g. if ``0.2`` randomize delays between + ``delay*0.8`` and ``delay*1.2``. - # Warn when delayed requests exceed this threshold - THROTTLING_DELAYED_REQUESTS_WARN_THRESHOLD = 500 + It can be set to a 2-item list with low and high factors, e.g. + ``[-0.1, 0.3]`` to randomize delays between ``delay*0.9`` and + ``delay*1.3``. -Integration with Direct Downloads ---------------------------------- + If ``True``, ``0.5`` (i.e. ±50%) is used as the randomization factor. If + ``False``, no randomization is applied. -The throttling system also works with direct downloads made via ``crawler.engine.download()``: -.. code-block:: python +.. _throttling-api: - # In a pipeline or extension - @inlineCallbacks - def process_item(self, item, spider): - # This request will respect throttling scopes - request = scrapy.Request(item["image_url"]) - response = yield spider.crawler.engine.download(request) - # Process response... +API +=== -Best Practices -============== +.. autoclass:: scrapy.throttling.ThrottlingManagerProtocol + :members: + :member-order: bysource -Choosing the Right Approach ----------------------------- +.. autoclass:: scrapy.throttling.ThrottlingManager + :members: get_response_delay -1. **Start Simple**: Begin with :setting:`CONCURRENT_REQUESTS` and :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -2. **Add Delays When Needed**: Use :setting:`DOWNLOAD_DELAY` for frequency-sensitive sites -3. **Use Custom Slots for Special Cases**: When different parts of a site need different treatment -4. **Advanced Buckets for Complex APIs**: When dealing with modern APIs with sophisticated rate limiting +.. autoclass:: scrapy.throttling.ThrottlingScopeManagerProtocol + :members: + :member-order: bysource -Respectful Scraping -------------------- +.. autoclass:: scrapy.throttling.ThrottlingScopeManager -- Always check ``robots.txt`` and respect ``Crawl-delay`` directives -- Monitor server response times and adjust settings if you're causing delays -- Watch for 429, 503, and other error responses that indicate you're going too fast -- Consider the server's perspective: your efficiency shouldn't come at their expense +.. autofunction:: scrapy.throttling.scope_cache +.. autofunction:: scrapy.throttling.add_scope +.. autofunction:: scrapy.throttling.update_scope_backoff -Common Patterns ---------------- -**E-commerce Site**: -.. code-block:: python - - CONCURRENT_REQUESTS_PER_DOMAIN = 2 - DOWNLOAD_DELAY = 1 - RANDOMIZE_DOWNLOAD_DELAY = True - -**REST API**: - -.. code-block:: python - - # Use throttling scopes to respect API rate limits - THROTTLING_ENABLED = True - CONCURRENT_REQUESTS_PER_DOMAIN = 5 - -**Mixed Content (API + Web)**: - -.. code-block:: python - - DOWNLOAD_SLOTS = { - "api": {"concurrency": 10, "delay": 0.1}, - "web": {"concurrency": 2, "delay": 2.0}, - } - -Monitoring and Debugging -======================== - -Scrapy provides several ways to monitor your throttling: - -.. code-block:: python - - # Enable autothrottle debugging (for legacy autothrottle extension only) - AUTOTHROTTLE_DEBUG = True - - # Monitor stats - # Check spider.crawler.stats for throttling-related statistics - -.. warning:: - The AutoThrottle extension is deprecated and not recommended for new projects. It uses a simplistic latency-based approach that doesn't align with modern server throttling patterns. Use the throttling scope system instead. - -Settings Reference -================== - -Global Settings ---------------- - -RANDOMIZE_DOWNLOAD_DELAY - -**Default**: ``True`` - -If enabled, Scrapy will wait a random amount of time (between 0.5 * and 1.5 * ``DOWNLOAD_DELAY``) while fetching requests from the same domain. - -Slot Settings -------------- - -DOWNLOAD_SLOTS - -**Default**: ``{}`` - -A dictionary containing the download slots and their settings. Each slot can have the following settings: - -* ``concurrency`` - Maximum concurrent requests for this slot -* ``delay`` - Download delay for this slot (in seconds) - -Advanced Throttling Settings ----------------------------- - -.. setting:: THROTTLING_ENABLED - -**Default**: ``False`` - -Enable the advanced throttling scope system. - -.. setting:: THROTTLING_MAX_DELAYED_REQUESTS - -**Default**: ``1000`` - -Maximum number of delayed requests to keep in memory. - -.. setting:: THROTTLING_DELAYED_REQUESTS_WARN_THRESHOLD - -**Default**: ``500`` - -Warn when the number of delayed requests exceeds this threshold. .. - TODO: Provide real-life examples of throttling configurations, including - exception and error response handling, throttling based on responses, - delay adjustment geared towards rate limits optimization, querying of - external resources for throttling decisions, etc. + When a throttling scope is configured with a **concurrency higher than 1**, + backoff is handled separately per slot. If at some point all + slots reach the maximum backoff delay, a “concurrency backoff” + starts, controlled by the following setting: + - .. setting:: BACKOFF_CONCURRENCY_DECREASE_FACTOR + :setting:`BACKOFF_CONCURRENCY_FACTOR` (default: ``0.5``) + The factor by which the concurrency is decreased during concurrency + backoff. -- .. setting:: CONCURRENT_REQUESTS + .. _scope-backoff: - ``CONCURRENT_REQUESTS`` (default: ``16``): The maximum number of total - concurrent requests. + Backoff settings can be overridden per throttling scope using + :setting:`THROTTLING_SCOPES`: + + .. code-block:: python + + { + "example.com": { + "backoff": { + "http_codes": [429, 503], + "exceptions": ["builtins.IOError"], + "delay_factor": 1.2, + "max_delay": 180.0, + "min_delay": 5.0, + "jitter": [0.01, 0.33], + "concurrency_factor": 0.8, + } + }, + } .. - TODO: Since this setting is more about limiting spider-side resources than - throttling, maybe it does not need to be covered in this page. - - -.. - Implement a signal that can be emitted to change the throttling config of - a specific throttling scope. - - -.. - TODO: Explain how things work, for every setting/parameter, when a request - is assigned multiple scopes. - -.. - TODO: Explain how scope units work, and give usage examples. + TODO: Continue from here .. TODO: Support backoff in THROTTLING_SCOPES having a cls key with a class @@ -924,3 +784,275 @@ Warn when the number of delayed requests exceeds this threshold. .. TODO: Make sure that the API is flexible enough to support scrapy-zyte-api use cases. + +.. + TODO: Try to figure out the implementation for the entire feature, and how + that may affect the user-facing API. + +.. + TODO: Think about: + - How backoff state is handled. + - Implement a setting that allows customizing the backoff window, which + should be 60.0 seconds by default, and should be overridable per scope. + - How backoff could be fine-tuned to deal with scenarios where it is + ping-poning between 2 states, and it should be possible to go for an + in-between state that allows maximizing throughtput while still + minimizing backoff feedback (backoff response or exception), aiming for a + single backoff feedback per backoff window. + +.. + TODO: Figure out the interaction APIs between components when we have: + - The scheduler using the throttling manager to decide whether a request + can be sent or not, and to sort requests. + +.. + TODO: Review all related issues and PRs, including those about multiple + slots, about per-request delays, and about politeness, and make sure every + scenario is covered here. + +.. + TODO: See if there is any info from the old autothrottle docs worth + keeping: + + .. _topics-autothrottle: + + ====================== + AutoThrottle extension + ====================== + + This is an extension for automatically throttling crawling speed based on load + of both the Scrapy server and the website you are crawling. + + Design goals + ============ + + 1. be nicer to sites instead of using default download delay of zero + 2. automatically adjust Scrapy to the optimum crawling speed, so the user + doesn't have to tune the download delays to find the optimum one. + The user only needs to specify the maximum concurrent requests + it allows, and the extension does the rest. + + .. _autothrottle-algorithm: + + How it works + ============ + + Scrapy allows defining the concurrency and delay of different download slots, + e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are + assigned to slots based on their URL domain, although it is possible to + customize the download slot of any request. + + The AutoThrottle extension adjusts the delay of each download slot dynamically, + to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent + requests on average to each remote website. + + It uses download latency to compute the delays. The main idea is the + following: if a server needs ``latency`` seconds to respond, a client + should send a request each ``latency/N`` seconds to have ``N`` requests + processed in parallel. + + Instead of adjusting the delays one can just set a small fixed + download delay and impose hard limits on concurrency using + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or + :setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar + effect, but there are some important differences: + + * because the download delay is small there will be occasional bursts + of requests; + * often non-200 (error) responses can be returned faster than regular + responses, so with a small download delay and a hard concurrency limit + crawler will be sending requests to server faster when server starts to + return errors. But this is an opposite of what crawler should do - in case + of errors it makes more sense to slow down: these errors may be caused by + the high request rate. + + AutoThrottle doesn't have these issues. + + Throttling algorithm + ==================== + + AutoThrottle algorithm adjusts download delays based on the following rules: + + 1. spiders always start with a download delay of + :setting:`AUTOTHROTTLE_START_DELAY`; + 2. when a response is received, the target download delay is calculated as + ``latency / N`` where ``latency`` is a latency of the response, + and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`. + 3. download delay for next requests is set to the average of previous + download delay and the target download delay; + 4. latencies of non-200 responses are not allowed to decrease the delay; + 5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater + than :setting:`AUTOTHROTTLE_MAX_DELAY` + + .. note:: The AutoThrottle extension honours the standard Scrapy settings for + concurrency and delay. This means that it will respect + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`CONCURRENT_REQUESTS_PER_IP` options and + never set a download delay lower than :setting:`DOWNLOAD_DELAY`. + + .. _download-latency: + + In Scrapy, the download latency is measured as the time elapsed between + establishing the TCP connection and receiving the HTTP headers. + + Note that these latencies are very hard to measure accurately in a cooperative + multitasking environment because Scrapy may be busy processing a spider + callback, for example, and unable to attend downloads. However, these latencies + should still give a reasonable estimate of how busy Scrapy (and ultimately, the + server) is, and this extension builds on that premise. + + .. reqmeta:: autothrottle_dont_adjust_delay + + Prevent specific requests from triggering slot delay adjustments + ================================================================ + + AutoThrottle adjusts the delay of download slots based on the latencies of + responses that belong to that download slot. The only exceptions are non-200 + responses, which are only taken into account to increase that delay, but + ignored if they would decrease that delay. + + You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to + ``True`` in any request to prevent its response latency from impacting the + delay of its download slot: + + .. code-block:: python + + from scrapy import Request + + Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) + + Note, however, that AutoThrottle still determines the starting delay of every + download slot by setting the ``download_delay`` attribute on the running + spider. If you want AutoThrottle not to impact a download slot at all, in + addition to setting this meta key in all requests that use that download slot, + you might want to set a custom value for the ``delay`` attribute of that + download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. + + Settings + ======== + + The settings used to control the AutoThrottle extension are: + + * :setting:`AUTOTHROTTLE_ENABLED` + * :setting:`AUTOTHROTTLE_START_DELAY` + * :setting:`AUTOTHROTTLE_MAX_DELAY` + * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` + * :setting:`AUTOTHROTTLE_DEBUG` + * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + * :setting:`CONCURRENT_REQUESTS_PER_IP` + * :setting:`DOWNLOAD_DELAY` + + For more information see :ref:`autothrottle-algorithm`. + + .. setting:: AUTOTHROTTLE_ENABLED + + AUTOTHROTTLE_ENABLED + ~~~~~~~~~~~~~~~~~~~~ + + Default: ``False`` + + Enables the AutoThrottle extension. + + .. setting:: AUTOTHROTTLE_START_DELAY + + AUTOTHROTTLE_START_DELAY + ~~~~~~~~~~~~~~~~~~~~~~~~ + + Default: ``5.0`` + + The initial download delay (in seconds). + + .. setting:: AUTOTHROTTLE_MAX_DELAY + + AUTOTHROTTLE_MAX_DELAY + ~~~~~~~~~~~~~~~~~~~~~~ + + Default: ``60.0`` + + The maximum download delay (in seconds) to be set in case of high latencies. + + .. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY + + AUTOTHROTTLE_TARGET_CONCURRENCY + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Default: ``1.0`` + + Average number of requests Scrapy should be sending in parallel to remote + websites. It must be higher than ``0.0``. + + By default, AutoThrottle adjusts the delay to send a single + concurrent request to each of the remote websites. Set this option to + a higher value (e.g. ``2.0``) to increase the throughput and the load on remote + servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value + (e.g. ``0.5``) makes the crawler more conservative and polite. + + Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected + when AutoThrottle extension is enabled. This means that if + ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or + :setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number + of concurrent requests. + + At every given time point Scrapy can be sending more or less concurrent + requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested + value the crawler tries to approach, not a hard limit. + + .. setting:: AUTOTHROTTLE_DEBUG + + AUTOTHROTTLE_DEBUG + ~~~~~~~~~~~~~~~~~~ + + Default: ``False`` + + Enable AutoThrottle debug mode which will display stats on every response + received, so you can see how the throttling parameters are being adjusted in + real time. + +.. + TODO: Figure out how to properly deprecate AutoThrottle settings, API and + keep its presence in the list of enabled extensions without that triggering + a deprecation warning. + +.. + TODO: Avoid so much code duplication in add_scope and update_scope_backoff. + +.. + TODO: Describe in detail the backoff algorithm, with steps, etc., and also + covering how it tries to maintain a sweet spot avoiding + +.. + TODO: Define typed dicts for all dicts supported by settings, e.g. for + THROTTLING_SCOPES. + +.. + TODO: Provide an example implementation of a custom throttling scope + manager based on the current implementation of scrapy-zyte-api retrying + policy. + +.. + TODO: If get_scopes reports expected quotas, treat those as actual + consumptions at run time, and then handle the actual consumptions reported + in get_response_backoff by reporting the difference. + + e.g. if a request is expected to consume 2.0 quota, lower the available + quota in the current window by 2.0. If the response then reports that 2.5 + was actually consumed, then report 0.5 as the difference, to be also + substracted from the current window. + +.. + TODO: Implement some kind of system to free scope-related memory if a given + scope is not used for a while? + +.. + TODO: Think about tracking effective concurrency and delays, and reporting + to users when their custom settings are not effective for a significant + period of time. + +.. + TODO: Provide a complete list of keys that THROTTLING_SCOPES supports. + +.. + TODO: Update news.rst with all the changes, but only once everything else + has been written. Make sure to include the new REDIRECT_MAX_DELAY setting. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d6fb9972e..d3110b488 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -47,6 +47,7 @@ if TYPE_CHECKING: from scrapy.logformatter import LogFormatter from scrapy.statscollectors import StatsCollector + from scrapy.throttling import ThrottlingManagerProtocol from scrapy.utils.request import RequestFingerprinterProtocol @@ -84,6 +85,7 @@ class Crawler: self.stats: StatsCollector | None = None self.logformatter: LogFormatter | None = None self.request_fingerprinter: RequestFingerprinterProtocol | None = None + self.throttler: ThrottlingManagerProtocol | None = None self.spider: Spider | None = None self.engine: ExecutionEngine | None = None @@ -113,6 +115,10 @@ class Crawler: load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), self, ) + self.throttler = build_from_crawler( + load_object(self.settings["THROTTLING_MANAGER"]), + self, + ) reactor_class: str = self.settings["TWISTED_REACTOR"] event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"] diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index cdb0671ae..89fa6ff67 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -2,9 +2,10 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING +from warnings import warn from scrapy import Request, Spider, signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -24,6 +25,14 @@ class AutoThrottle: if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"): raise NotConfigured + warn( + "You have set the AUTOTHROTTLE_ENABLED setting to True, however " + "the AutoThrottle extension is deprecated; use throttling and " + "backoff settings instead: " + "https://docs.scrapy.org/en/latest/topics/throttling.html", + ScrapyDeprecationWarning, + ) + self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") self.target_concurrency: float = crawler.settings.getfloat( "AUTOTHROTTLE_TARGET_CONCURRENCY" diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0432a7231..e1ebdf707 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,87 +1,13 @@ -# Scrapy settings for $project_name project -# -# For simplicity, this file contains only settings considered important or -# commonly used. You can find more settings consulting the documentation: -# -# https://docs.scrapy.org/en/latest/topics/settings.html -# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -# https://docs.scrapy.org/en/latest/topics/spider-middleware.html +# https://docs.scrapy.org/en/latest/topics/settings.html BOT_NAME = "$project_name" SPIDER_MODULES = ["$project_name.spiders"] NEWSPIDER_MODULE = "$project_name.spiders" -ADDONS = {} +# Crawl responsibly by identifying yourself (and your website) through the +# User-Agent header: +#USER_AGENT = "$project_name (+https://your-domain.example)" - -# Crawl responsibly by identifying yourself (and your website) on the user-agent -#USER_AGENT = "$project_name (+http://www.yourdomain.com)" - -# Obey robots.txt rules -ROBOTSTXT_OBEY = True - -# Concurrency and throttling settings -#CONCURRENT_REQUESTS = 16 -CONCURRENT_REQUESTS_PER_DOMAIN = 1 -DOWNLOAD_DELAY = 1 - -# Disable cookies (enabled by default) -#COOKIES_ENABLED = False - -# Disable Telnet Console (enabled by default) -#TELNETCONSOLE_ENABLED = False - -# Override the default request headers: -#DEFAULT_REQUEST_HEADERS = { -# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", -# "Accept-Language": "en", -#} - -# Enable or disable spider middlewares -# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html -#SPIDER_MIDDLEWARES = { -# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543, -#} - -# Enable or disable downloader middlewares -# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html -#DOWNLOADER_MIDDLEWARES = { -# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543, -#} - -# Enable or disable extensions -# See https://docs.scrapy.org/en/latest/topics/extensions.html -#EXTENSIONS = { -# "scrapy.extensions.telnet.TelnetConsole": None, -#} - -# Configure item pipelines -# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html -#ITEM_PIPELINES = { -# "$project_name.pipelines.${ProjectName}Pipeline": 300, -#} - -# Enable and configure the AutoThrottle extension (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/autothrottle.html -#AUTOTHROTTLE_ENABLED = True -# The initial download delay -#AUTOTHROTTLE_START_DELAY = 5 -# The maximum download delay to be set in case of high latencies -#AUTOTHROTTLE_MAX_DELAY = 60 -# The average number of requests Scrapy should be sending in parallel to -# each remote server -#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 -# Enable showing throttling stats for every response received: -#AUTOTHROTTLE_DEBUG = False - -# Enable and configure HTTP caching (disabled by default) -# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings -#HTTPCACHE_ENABLED = True -#HTTPCACHE_EXPIRATION_SECS = 0 -#HTTPCACHE_DIR = "httpcache" -#HTTPCACHE_IGNORE_HTTP_CODES = [] -#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" - -# Set settings whose default value is deprecated to a future-proof value +# Set settings whose default value is deprecated to a future-proof value: FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 1d57e1a67..1f5168cd5 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1,22 +1,190 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Protocol +import datetime as dt +from collections.abc import Awaitable, Iterable +from datetime import UTC +from email.utils import parsedate_to_datetime +from functools import wraps +from typing import TYPE_CHECKING, Any, Callable, Protocol, TypedDict, Union +from weakref import WeakKeyDictionary +from typing_extensions import NotRequired, Self + +from scrapy.http import Request, Response from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import load_object if TYPE_CHECKING: - from collections.abc import Iterable + from scrapy.crawler import Crawler - from scrapy.http import Request, Response + +def _parse_retry_after(response: Response) -> float | None: + value = response.headers.get("Retry-After") + if not value: + return None + try: + value = value.decode("utf-8").strip() + except UnicodeDecodeError: + return None + if value.isdigit(): + return float(value) # seconds + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + return None + if date.tzinfo is None: + date = date.replace(tzinfo=UTC) + now = dt.datetime.now(UTC) + seconds_to_wait = (date - now).total_seconds() + return max(0, int(seconds_to_wait)) or None + + +def _parse_ratelimit_reset(response: Response) -> float | None: + value = response.headers.get("RateLimit-Reset") + if not value: + return None + try: + value = value.decode("utf-8").strip() + except UnicodeDecodeError: + return None + try: + return float(value) + except ValueError: + return None + + +class BackoffScopeData(TypedDict): + delay: NotRequired[float] + consumed: NotRequired[float] + remaining: NotRequired[float] + + +ScopeID = str +BackoffData = Union[None, ScopeID, Iterable[ScopeID], dict[ScopeID, BackoffScopeData]] +RequestScopes = Union[None, ScopeID, Iterable[ScopeID], dict[ScopeID, float | None]] + + +def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: + if scopes is None: + return () + if isinstance(scopes, str): + return (scopes,) + if isinstance(scopes, dict): + return scopes.keys() + return iter(scopes) + + +def add_scope( + scopes: RequestScopes, + scope: ScopeID, + value: float | None = None, + /, +) -> RequestScopes: + """Add *scope* to *scopes* with *value*. + + This is a utility function to help extending the output of + :meth:`~ThrottlingManagerProtocol.get_scopes`, e.g. in + :class:`ThrottlingManager` subclasses. + """ + if value is not None: + if not isinstance(scopes, dict): + if scopes is None: + scopes = {} + elif isinstance(scopes, str): + scopes = {scopes: None} + elif isinstance(scopes, Iterable): + scopes = {s: None for s in scopes} + else: + raise TypeError( + f"Invalid type ({type(scopes)}) of scopes value " + f"{scopes!r}. Expected None, str, Iterable or dict." + ) + if scope in scopes and not isinstance(scopes[scope], dict): + raise TypeError(f"Scope {scope!r} has a non-dict value in {scopes!r}") + scopes[scope] = value + elif scopes is None: + scopes = scope + elif isinstance(scopes, str): + if scopes != scope: + scopes = {scopes, scope} + elif isinstance(scopes, dict): + if scope not in scopes: + scopes[scope] = None + elif isinstance(scopes, Iterable): + if scope not in scopes: + scopes = set(scopes) | {scope} + else: + raise TypeError( + f"Invalid type ({type(scopes)}) of scopes value " + f"{scopes!r}. Expected None, str, Iterable or dict." + ) + return scopes + + +def update_scope_backoff( + backoff: BackoffData, + scope: ScopeID, + /, + *, + delay: float | None = None, + consumed: float | None = None, +) -> BackoffData: + """Add *scope* to *backoff* or update its existing entry the given + parameters. + + This is a utility function to help extending the output of + :meth:`~ThrottlingManagerProtocol.get_initial_backoff`, + :meth:`~ThrottlingManagerProtocol.get_response_backoff` or + :meth:`~ThrottlingManagerProtocol.get_exception_backoff`, e.g. in + :class:`ThrottlingManager` subclasses. + """ + has_params = delay is not None or consumed is not None + if has_params: + if not isinstance(backoff, dict): + if backoff is None: + backoff = {} + elif isinstance(backoff, str): + backoff = {backoff: {}} + elif isinstance(backoff, Iterable): + backoff = {s: {} for s in backoff} + else: + raise TypeError( + f"Invalid type ({type(backoff)}) of scopes value " + f"{backoff!r}. Expected None, str, Iterable or dict." + ) + if scope in backoff: + if not isinstance(backoff[scope], dict): + raise TypeError(f"Scope {scope!r} has a non-dict value in {backoff!r}") + else: + backoff[scope] = {} + if delay is not None: + backoff[scope]["delay"] = delay + if consumed is not None: + backoff[scope]["consumed"] = consumed + elif backoff is None: + backoff = scope + elif isinstance(backoff, str): + if backoff != scope: + backoff = {backoff, scope} + elif isinstance(backoff, dict): + if scope not in backoff: + backoff[scope] = {} + elif isinstance(backoff, Iterable): + if scope not in backoff: + backoff = set(backoff) | {scope} + else: + raise TypeError( + f"Invalid type ({type(backoff)}) of scopes value " + f"{backoff!r}. Expected None, str, Iterable or dict." + ) + return backoff class ThrottlingManagerProtocol(Protocol): """A protocol for :setting:`THROTTLING_MANAGER` :ref:`components `.""" - def get_scopes( - self, request: Request - ) -> None | str | Iterable[str] | dict[str, float]: + async def get_scopes(self, request: Request) -> RequestScopes: """Return the :ref:`throttling scopes ` that apply to *request*. @@ -25,29 +193,39 @@ class ThrottlingManagerProtocol(Protocol): keys and :ref:`throttling quotas ` as values. """ - def get_response_throttling( - self, response: Response - ) -> None | str | Iterable[str] | dict[str, dict[str, Any]]: - """Return a throttling data update based on *response*. + async def get_initial_backoff(self) -> BackoffData: + """Return the initial throttling data. - Return ``None`` if there is nothing new to report, i.e. the response is - not a :ref:`backoff ` response. + This method is called before the first request is sent, and it should + be used to provide an initial throttling state, to be used before it is + updated with later calls to :meth:`get_response_backoff` and + :meth:`get_exception_backoff`. - If the response indicates that one or more scopes are currently - exhausted, return a string for a single scope or an iterable of strings - for multiple scopes. + **Return values:** - If the response indicates any other information about one or more - scopes, return a dict with scopes as keys and dict values. Dict values - support the following keys: + You may return any of the following: - - ``"delay"``: a float indicating how many seconds to wait before - sending another request for the scope. + - ``None``: no throttling data to report. - - ``"quota"``: a float indicating the remaining :ref:`throttling - quota `. + - A string: a single scope name, indicating that the scope is + currently exhausted. - If ``"quota"`` is not specified, the resource is considered exhausted. + - An iterable of strings: multiple scope names, indicating that + those scopes are currently exhausted. + + - A dict with scope names as keys and dict values. Dict values + support the following keys: + + - ``"delay"``: a float indicating how many seconds to wait before + sending another request for the scope. + + - ``"quota"``: a float indicating the remaining :ref:`throttling + quota `. + + If ``"quota"`` is not specified, the resource is considered + exhausted. + + For example: .. code-block:: python @@ -58,16 +236,63 @@ class ThrottlingManagerProtocol(Protocol): } """ - def get_exception_throttling( + async def get_response_backoff(self, response: Response) -> BackoffData: + """Return a throttling data update based on *response*. + + It supports the same return values as :meth:`get_initial_backoff`. + """ + + async def get_exception_backoff( self, request: Request, exception: Exception - ) -> None | str | Iterable[str] | dict[str, dict[str, Any]]: + ) -> BackoffData: """Return a throttling data update based on *exception* and the *request* that caused it. - It supports the same return values as :meth:`get_response_throttling`. + It supports the same return values as :meth:`get_initial_backoff`. """ +GetScopesMethod = Callable[ + [ThrottlingManagerProtocol, Request], Awaitable[RequestScopes] +] + + +def scope_cache(f: GetScopesMethod) -> GetScopesMethod: + """Decorator to cache the result of + :meth:`~ThrottlingManagerProtocol.get_scopes` calls. + + It should be used so that calls to + :meth:`~ThrottlingManagerProtocol.get_scopes` from methods like + :meth:`~ThrottlingManagerProtocol.get_response_backoff` or + :meth:`~ThrottlingManagerProtocol.get_exception_backoff` do not become + unnecessarily expensive. + + For example: + + .. code-block:: python + + from scrapy.utils.httpobj import urlparse_cached + from scrapy.utils.throttling import scope_cache + + + class MyThrottlingManager: + @scope_cache + async def get_scopes(self, request): + return urlparse_cached(request).netloc + """ + cache = WeakKeyDictionary() + + @wraps(f) + async def wrapper(self, request: Request): + if request in cache: + return cache[request] + scopes = await f(self, request) + cache[request] = scopes + return scopes + + return wrapper + + class ThrottlingManager: """The default :setting:`THROTTLING_MANAGER` class. @@ -75,7 +300,99 @@ class ThrottlingManager: backoff according to :ref:`backoff settings `. """ - def get_scopes( - self, request: Request - ) -> None | str | Iterable[str] | dict[str, float]: + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def __init__(self, crawler: Crawler) -> None: + self.crawler = crawler + self.throttler = crawler.throttler + self.backoff_http_codes = set(crawler.settings.getlist("BACKOFF_HTTP_CODES")) + self.backoff_exceptions = tuple( + load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS") + ) + + @scope_cache + async def get_scopes( + self: ThrottlingManagerProtocol, request: Request + ) -> RequestScopes: return urlparse_cached(request).netloc + + async def get_initial_backoff(self) -> BackoffData: + return None + + async def get_response_backoff(self, response: Response) -> BackoffData: + if response.status not in self.backoff_http_codes: + return None + assert response.request is not None + assert self.throttler is not None + scopes = await self.throttler.get_scopes(response.request) + if delay := self.get_response_delay(response): + scopes = {scope: {"delay": delay} for scope in iter_scopes(scopes)} + return scopes + + def get_response_delay(self, response: Response) -> float | None: + """Return the throttling delay requested by the response.""" + retry_after = _parse_retry_after(response) + ratelimit_reset = _parse_ratelimit_reset(response) + if retry_after is None and ratelimit_reset is None: + return None + if retry_after is not None and ratelimit_reset is not None: + return max(retry_after, ratelimit_reset) + if retry_after is not None: + return retry_after + assert ratelimit_reset is not None + return ratelimit_reset + + async def get_exception_backoff( + self, request: Request, exception: Exception + ) -> BackoffData: + if isinstance(exception, self.backoff_exceptions): + assert self.throttler is not None + return await self.throttler.get_scopes(request) + return None + + +class ThrottlingScopeManagerProtocol(Protocol): + """A protocol for :setting:`THROTTLING_SCOPE_MANAGER` :ref:`components + `. + + The ``__init__`` method gets a ``config`` dict with the base configuration + of the managed throttling scope. For example: + + .. code-block:: python + + { + "id": "example.com", + "concurrency": 1.0, + "delay": 1.0, + "jitter": 0.5, + "quota": 1000.0, + "window": 60.0, + "backoff": { + "http_codes": [429, 503], + "exceptions": ["builtins.IOError"], + "delay_factor": 1.2, + "max_delay": 180.0, + "min_delay": 5.0, + "jitter": [0.01, 0.33], + "concurrency_factor": 0.8, + }, + "rampup": { + "backoff_target": 1, + "delay_factor": 0.8, + "min_delay": 0.05, + }, + } + """ + + @classmethod + def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self: + return cls(crawler, config) + + def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: + pass + + +class ThrottlingScopeManager: + """The default :setting:`THROTTLING_SCOPE_MANAGER` class.""" From b21e654800c1e1b63613c459ddcb7e8c9e09f798 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 15 Jun 2026 14:10:54 +0200 Subject: [PATCH 04/55] Align default values in docs with the main branch --- docs/topics/throttling.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index b3bc7e3de..f376b135d 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,7 +26,7 @@ The main throttling :ref:`settings ` are: - .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1``) + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` (:ref:`fallback `: ``8``)) Maximum number of simultaneous requests per domain. @@ -36,7 +36,7 @@ The main throttling :ref:`settings ` are: - .. setting:: DOWNLOAD_DELAY - :setting:`DOWNLOAD_DELAY` (default: ``1.0``) + :setting:`DOWNLOAD_DELAY` (default: ``1`` (:ref:`fallback `: ``0``)) Minimum seconds between any two requests to the same domain. From b6bc947cce8280d8a6f800650d41e01d698a11b2 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 24 Jun 2026 10:37:42 +0200 Subject: [PATCH 05/55] =?UTF-8?q?scrapy/addons.py=20=E2=86=92=20scrapy/add?= =?UTF-8?q?ons/=5F=5Finit=5F=5F.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/{addons.py => addons/__init__.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scrapy/{addons.py => addons/__init__.py} (100%) diff --git a/scrapy/addons.py b/scrapy/addons/__init__.py similarity index 100% rename from scrapy/addons.py rename to scrapy/addons/__init__.py From d8d519367d0e99d23e19adacae411f3b60e7837d Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 24 Jun 2026 10:57:56 +0200 Subject: [PATCH 06/55] Fix docs/requirements.txt --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index eb0436771..655b45e72 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -134,6 +134,7 @@ sphinx==9.1.0 # sphinx-llms-txt # sphinx-markdown-builder # sphinx-notfound-page + # sphinx-reredirects # sphinx-rtd-theme # sphinx-scrapy # sphinxcontrib-jquery @@ -147,7 +148,7 @@ sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builde # via sphinx-scrapy sphinx-notfound-page==1.1.0 # via -r docs/requirements.in -sphinx-reredirects==1.0.0 +sphinx-reredirects==1.1.0 # via -r docs/requirements.in sphinx-rtd-dark-mode==1.3.0 # via -r docs/requirements.in From d18085dc3efe348b9eaca0410d3e311576a70f03 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 24 Jun 2026 11:49:51 +0200 Subject: [PATCH 07/55] Update docs --- docs/topics/settings.rst | 1 + docs/topics/throttling.rst | 120 +++++++++++++++++++++++-------------- 2 files changed, 77 insertions(+), 44 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 405805133..1c875d4d3 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1917,6 +1917,7 @@ See :ref:`asyncio-without-reactor` for more information about this mode. .. versionadded:: 2.15.0 + .. setting:: TWISTED_REACTOR TWISTED_REACTOR diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index f376b135d..9b4337089 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -4,6 +4,8 @@ Throttling ========== +.. versionadded:: VERSION + Sending too many requests too quickly can `overwhelm websites`_. :ref:`Throttling ` and :ref:`backoff ` aim to prevent that. @@ -45,13 +47,17 @@ The main throttling :ref:`settings ` are: - .. setting:: DOWNLOAD_DELAY_PER_SLOT - :setting:`DOWNLOAD_DELAY_PER_SLOT` (default: ``1.0``) + :setting:`DOWNLOAD_DELAY_PER_SLOT` (default: ``None``) - Minimum seconds between requests in the same slot. + Minimum seconds to wait between two consecutive requests sent to the same + download slot. Unlike :setting:`DOWNLOAD_DELAY`, which applies per domain + (:ref:`throttling scope `), this delay is per slot. - If a slot sends a request and receives its response before this delay has - elapsed, it must wait before sending the next request. The wait time is - measured from when the previous request was sent. + When ``None`` (default), the per-slot delay falls back to + :setting:`DOWNLOAD_DELAY`, preserving the historical behavior where + :setting:`DOWNLOAD_DELAY` was enforced per slot. + + The wait time is measured from when the previous request was sent. For example, with ``CONCURRENT_REQUESTS_PER_DOMAIN = 2``, ``DOWNLOAD_DELAY = 0.3``, and ``DOWNLOAD_DELAY_PER_SLOT = 1.0``, sending 3 requests to the same domain @@ -79,13 +85,17 @@ When configuring these settings, note that: .. setting:: THROTTLING_SCOPES + .. _per-domain-throttling: Per-domain throttling ===================== -The :setting:`THROTTLING_SCOPES` setting allows you to customize throttling behavior -for specific domains [1]_. +The :setting:`THROTTLING_SCOPES` setting allows you to customize throttling +behavior for specific domains [1]_. + +It is a dict that maps scope names to +:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. Its default value allows faster crawling of the testing website using during the :ref:`tutorial ` while maintaining conservative defaults @@ -118,19 +128,22 @@ The key settings are: :setting:`BACKOFF_HTTP_CODES` (default: ``[429, 502, 503, 504, 520, 521, 522, 523, 524]``) - HTTP status codes that trigger backoff. + HTTP response status codes that trigger backoff. - .. setting:: BACKOFF_DELAY_FACTOR :setting:`BACKOFF_DELAY_FACTOR` (default: ``2.0``) - Each backoff multiplies delay by this factor (2x, 4x, 8x, etc.). + Factor by which the delay of a scope is multiplied on each backoff step + (2×, 4×, 8×, etc.). - .. setting:: BACKOFF_MAX_DELAY :setting:`BACKOFF_MAX_DELAY` (default: ``300.0``) - Maximum delay cap to prevent excessively long waits. + Maximum delay, in seconds, that backoff can reach. Also caps + :ref:`Retry-After ` and :ref:`RateLimit-Reset + ` delays. .. _rampup: @@ -158,8 +171,9 @@ setting: :setting:`RAMPUP_BACKOFF_TARGET` (default: ``1``) - Target number of backoff responses per rampup window, indicating optimal - throughput. Can be a range like ``[1, 3]``. + Target number of backoff responses per :setting:`BACKOFF_WINDOW` that + :ref:`rampup ` aims for when probing the rate limit of a scope. + Can be a range like ``[1, 3]``. .. _retry-after: @@ -637,17 +651,12 @@ Additional settings Default: - .. code-block:: python + - :exc:`~scrapy.exceptions.DownloadFailedError` + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + - :exc:`~scrapy.exceptions.ResponseDataLossError` - [ - "twisted.internet.defer.TimeoutError", - "twisted.internet.error.TimeoutError", - "twisted.internet.error.TCPTimedOutError", - "twisted.web.client.ResponseFailed", - ] - - Exception classes that trigger backoff. Strings are interpreted as import - paths. + List of exception classes that trigger backoff when raised while + downloading a request. Strings are interpreted as import paths. .. seealso:: :setting:`RETRY_EXCEPTIONS` @@ -655,46 +664,45 @@ Additional settings :setting:`BACKOFF_JITTER` (default: ``0.1``) + Random jitter applied to each backoff delay, as a fraction of the delay. + With the default value of ``0.1`` the delay is randomized by ±10%. Overrides :setting:`RANDOMIZE_DOWNLOAD_DELAY` during backoff. - .. setting:: BACKOFF_MIN_DELAY :setting:`BACKOFF_MIN_DELAY` (default: ``1.0``) - Minimum delay during :ref:`backoff `. + Delay, in seconds, applied on the first backoff step (and the minimum + delay during backoff). - .. setting:: BACKOFF_WINDOW :setting:`BACKOFF_WINDOW` (default: ``60.0``) - During :ref:`backoff `, after a non-backoff response is received, - do not take the next step in backoff reduction until this amount of time - has passed and no new backoff feedback has been received. - - The number of seconds that need to pass since the last non-backoff response - without any other for the backoff to move towards the original throttling - configuration. + Time window, in seconds, used by :ref:`backoff ` and + :ref:`rampup `. During backoff, a :ref:`throttling scope + ` must go this many seconds without a new backoff + trigger (an HTTP error code from :setting:`BACKOFF_HTTP_CODES` or an + exception from :setting:`BACKOFF_EXCEPTIONS`) before its delay decreases + by one :setting:`BACKOFF_DELAY_FACTOR` step toward the configured value. + A new trigger resets the countdown. - .. setting:: DELAYED_REQUESTS_WARN_THRESHOLD :setting:`DELAYED_REQUESTS_WARN_THRESHOLD` (default: ``500``) - While throttled, requests in the :ref:`scheduler ` remain - in the scheduler. + Number of requests held back by :ref:`throttling ` at which + Scrapy logs a warning, to help detect throttling configurations that hold + back more requests than expected. - However, requests sent with :meth:`engine.download() - ` bypass the scheduler. This - includes requests sent by some built-in :ref:`components - ` and :ref:`inline requests `. - - When such requests are throttled, they are paused and kept in memory, along + While throttled, requests in the :ref:`scheduler ` + remain in the scheduler. However, requests sent with :meth:`engine.download() + ` bypass the scheduler, + including requests sent by some built-in :ref:`components + ` and :ref:`inline requests `. When + such requests are throttled, they are paused and kept in memory, along with any run time context from the code that is sending them. If they - accumulate, they can become a memory issue that may require you to rethink - your throttling parameters or crawl strategy. - - :setting:`DELAYED_REQUESTS_WARN_THRESHOLD` defines a threshold for such - requests. The first time that this many such requests are being throttled - at the same time, a warning is issued. + accumulate, they can become a memory issue. - .. setting:: RANDOMIZE_DOWNLOAD_DELAY @@ -710,6 +718,30 @@ Additional settings If ``True``, ``0.5`` (i.e. ±50%) is used as the randomization factor. If ``False``, no randomization is applied. +- .. setting:: TARGET_RPM + + :setting:`TARGET_RPM` (default: ``None``) + + Target number of requests per minute *per domain*. It has no effect on its + own; it is read by :class:`~scrapy.addons.throttling.TargetRPMAddon`, which + must be enabled explicitly. + +- .. setting:: THROTTLING_DEBUG + + :setting:`THROTTLING_DEBUG` (default: ``False``) + + Whether to log :ref:`throttling ` decisions (per-scope delays, + backoff steps and recoveries) for debugging. + +- .. setting:: THROTTLING_SCOPE_MAX_IDLE + + :setting:`THROTTLING_SCOPE_MAX_IDLE` (default: ``3600.0``) + + Seconds of inactivity after which the state of a :ref:`throttling scope + ` is evicted from memory to bound memory usage on + long-running crawls. Set to ``0`` to never evict. Scopes in active backoff + are never evicted. + .. _throttling-api: From c93e430ccbdacfa9468a805f75445c80b3b9ecff Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 24 Jun 2026 13:35:07 +0200 Subject: [PATCH 08/55] Complete docs --- docs/topics/throttling.rst | 611 +++++++++++++++++-------------------- 1 file changed, 279 insertions(+), 332 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 9b4337089..852b9f8e9 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -145,6 +145,74 @@ The key settings are: :ref:`Retry-After ` and :ref:`RateLimit-Reset ` delays. +See :ref:`throttling-settings` for :setting:`BACKOFF_EXCEPTIONS`, +:setting:`BACKOFF_JITTER`, :setting:`BACKOFF_MIN_DELAY` and +:setting:`BACKOFF_WINDOW`. + +.. _backoff-algorithm: + +How backoff works +----------------- + +Every :ref:`throttling scope ` keeps a current delay that +starts at its configured value (``0`` by default, or :setting:`DOWNLOAD_DELAY` +for the default per-domain scopes). + +A **backoff trigger** is a response whose status code is in +:setting:`BACKOFF_HTTP_CODES` or a download exception whose type is in +:setting:`BACKOFF_EXCEPTIONS`. On each trigger: + +#. If the response carries a :ref:`Retry-After or RateLimit-Reset + ` value, that value (capped at + :setting:`BACKOFF_MAX_DELAY`) is honored as a hard minimum: no request is + sent for the scope until it elapses. + +#. Otherwise the delay grows exponentially: + + .. code-block:: text + + delay = min(BACKOFF_MAX_DELAY, max(BACKOFF_MIN_DELAY, delay * BACKOFF_DELAY_FACTOR)) + + :setting:`BACKOFF_JITTER` is then applied so that requests that backed off + together do not retry in lockstep. + +**Recovery** is linear: after a scope goes a full :setting:`BACKOFF_WINDOW` +without a new trigger, its delay drops by one :setting:`BACKOFF_DELAY_FACTOR` +step toward the configured value, and keeps dropping one step per quiet window +until it is back to the configured value. A new trigger resets the countdown. + +This exponential-increase / linear-decrease pattern, similar to TCP congestion +control, makes a scope back off quickly when a server is unhappy and return to +full speed gradually once it recovers. To keep a scope hovering around a target +rate instead of repeatedly probing and backing off, enable :ref:`rampup +`. + +.. _per-scope-backoff: + +Per-scope backoff configuration +------------------------------- + +The global ``BACKOFF_*`` settings can be overridden per scope with the +``"backoff"`` key of a :setting:`THROTTLING_SCOPES` entry, an instance of +:class:`~scrapy.throttling.BackoffConfig`: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPES = { + "example.com": { + "backoff": { + "http_codes": [429, 503], + "exceptions": ["builtins.IOError"], + "delay_factor": 1.2, + "max_delay": 180.0, + "min_delay": 5.0, + "jitter": [0.01, 0.33], + }, + }, + } + +Any key left out falls back to the matching global ``BACKOFF_*`` setting. .. _rampup: @@ -175,6 +243,32 @@ setting: :ref:`rampup ` aims for when probing the rate limit of a scope. Can be a range like ``[1, 3]``. +For every :setting:`BACKOFF_WINDOW` that stays **below** +:setting:`RAMPUP_BACKOFF_TARGET` backoff triggers, rampup increases throughput +one step: it first lowers the delay, and once the delay reaches its minimum it +raises the concurrency limit above the ``"min_concurrency"`` floor of the +scope. Windows that hit the target hold the current rate, and windows that +exceed it let normal :ref:`backoff ` reduce the rate. The result is a +rate that converges on roughly :setting:`RAMPUP_BACKOFF_TARGET` rate-limit +responses per window — the most throughput a scope allows without being +penalized. + +Rampup behavior can be fine-tuned per scope by giving ``"rampup"`` a dict +instead of ``True``: + +.. code-block:: python + :caption: ``settings.py`` + + THROTTLING_SCOPES = { + "api.toscrape.com": { + "rampup": { + "backoff_target": [1, 3], # overrides RAMPUP_BACKOFF_TARGET + "delay_factor": 0.5, # multiply the delay by this on each ramp-up step + "min_delay": 0.05, # do not ramp the delay below this + }, + }, + } + .. _retry-after: .. _rate-limiting-headers: @@ -257,6 +351,35 @@ throttling for such requests: .. note:: These custom throttling groups persist through redirects. For redirect-aware throttling assignment, see :ref:`custom-throttling-scopes`. +.. reqmeta:: throttling_delay + +Delaying a single request +------------------------- + +To hold a single request for a fixed number of seconds before it is sent, +regardless of its scopes, set the ``throttling_delay`` request metadata key: + +.. code-block:: python + + Request("https://example.com/slow", meta={"throttling_delay": 5.0}) + +The delay is applied once, the first time the request reaches the throttling +gate. + +.. reqmeta:: throttling_dont_track + +Excluding a request from throttling state +----------------------------------------- + +Some requests (authentication flows, one-off API calls, file downloads) should +not influence throttling state even if they get a :setting:`BACKOFF_HTTP_CODES` +response or raise a :setting:`BACKOFF_EXCEPTIONS` exception. Set the +``throttling_dont_track`` request metadata key to ``True`` to process such a +request normally without letting its outcome trigger :ref:`backoff `: + +.. code-block:: python + + Request("https://example.com/login", meta={"throttling_dont_track": True}) .. _throttling-scopes: @@ -277,9 +400,51 @@ Customizing throttling scopes There are 2 ways to customize throttling scopes. +To **configure existing scopes**, use the :setting:`THROTTLING_SCOPES` setting. +Its keys are scope names and its values are +:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts, which accept the +following keys: + +``concurrency`` (:class:`int`) + Maximum number of concurrent requests for the scope. When unset, the + per-domain concurrency (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) applies + instead. + +``min_concurrency`` (:class:`int`) + Concurrency floor that :ref:`backoff ` and :ref:`rampup ` + never drop below. Defaults to ``1``. + +``delay`` (:class:`float`) + Minimum seconds between requests for the scope. Defaults to + :setting:`DOWNLOAD_DELAY`. + +``jitter`` (:class:`float` or 2-:class:`list`) + Per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. + +``quota`` (:class:`float`) + Maximum :ref:`quota ` consumed per ``window``. + +``window`` (:class:`float`) + Quota window in seconds. Defaults to :setting:`THROTTLING_WINDOW`. + +``rampup`` (:class:`bool` or :class:`dict`) + Enables :ref:`rampup ` for the scope. + +``backoff`` (:class:`~scrapy.throttling.BackoffConfig`) + Per-scope :ref:`backoff overrides `. + +``manager`` (:class:`str` or :class:`type`) + Import path or class of a :ref:`custom scope manager + ` for the scope. + +``ignore_robots_txt`` (:class:`bool`) + Silences the warning logged when this configuration is more aggressive than + a :ref:`robots.txt Crawl-delay `. + .. setting:: THROTTLING_MANAGER -For anything else, set :setting:`THROTTLING_MANAGER` (default: +To **change how scopes are assigned** (or anything beyond per-scope settings), +set :setting:`THROTTLING_MANAGER` (default: :class:`~scrapy.throttling.ThrottlingManager`) to a :ref:`component ` that implements the :class:`~scrapy.throttling.ThrottlingManagerProtocol` protocol (or its import @@ -375,6 +540,65 @@ setting: }, } +A scope manager only needs to implement the +:class:`~scrapy.throttling.ThrottlingScopeManagerProtocol`. For example, the +following manager enforces a fixed quota per time window without any delay or +exponential backoff, similar to a fixed-window rate limiter: + +.. code-block:: python + :caption: ``myproject/throttling.py`` + + import time + + + class FixedWindowScopeManager: + def __init__(self, crawler, config): + self.limit = config["quota"] + self.window = config.get("window", 60.0) + self.reset_at = None + self.used = 0.0 + self.active = 0 + + @classmethod + def from_crawler(cls, crawler, config): + return cls(crawler, config) + + def can_send(self, now=None, amount=None): + now = now if now is not None else time.monotonic() + if self.reset_at is not None and now >= self.reset_at: + self.reset_at, self.used = None, 0.0 + if self.used and self.used + (amount or 0) > self.limit: + return self.reset_at - now + return 0.0 + + def record_sent(self, now=None, amount=None): + now = now if now is not None else time.monotonic() + if self.reset_at is None: + self.reset_at = now + self.window + self.used += amount or 0 + self.active += 1 + + def record_done(self, now=None): + self.active = max(0, self.active - 1) + + def record_backoff(self, delay=None, now=None): + pass # this manager does not back off + + def reconcile_quota(self, consumed=None, remaining=None, now=None): + if remaining is not None: + self.used = max(0.0, self.limit - remaining) + elif consumed is not None: + self.used = max(0.0, self.used + consumed) + + def set_base_delay(self, delay): + pass + + def set_concurrency(self, concurrency): + pass + + def is_idle(self, now, max_idle): + return self.active == 0 + .. _throttling-examples: @@ -612,20 +836,23 @@ window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas return scopes return add_scope(scopes, "cost", estimate_request_cost(request)) - - Reports the actual cost during response parsing: + - Reconciles the estimated cost with the actual cost reported by the + response, so that the quota tracks real spending: .. code-block:: python - from scrapy.throttling import ThrottlingManager + from scrapy.throttling import ThrottlingManager, update_scope_backoff class MyThrottlingManager(ThrottlingManager): async def get_response_backoff(self, response): - scopes = await super().get_response_backoff(response) - if "cost" not in scopes: - return scopes - actual_cost = float(response.headers.get("X-Actual-Cost", b"0")) - return update_scope_backoff(scopes, "cost", consumed_quota=actual_cost) + backoff = await super().get_response_backoff(response) + if response.headers.get("X-Actual-Cost") is None: + return backoff + estimated = estimate_request_cost(response.request) + actual = float(response.headers[b"X-Actual-Cost"]) + # Report the difference between actual and estimated cost. + return update_scope_backoff(backoff, "cost", consumed=actual - estimated) - Use the :setting:`THROTTLING_SCOPES` setting to set a maximum cost per time window: @@ -742,6 +969,46 @@ Additional settings long-running crawls. Set to ``0`` to never evict. Scopes in active backoff are never evicted. +.. _autothrottle-migration: + +Migrating from AutoThrottle +=========================== + +The ``AutoThrottle`` extension is deprecated in favor of the throttling and +:ref:`backoff ` system described here, which is always active and does +not need to be enabled. + +Setting :setting:`AUTOTHROTTLE_ENABLED` to ``True`` still works but logs a +deprecation warning. To migrate, drop the ``AUTOTHROTTLE_*`` settings and use +the following equivalents: + +.. list-table:: + :header-rows: 1 + + * - AutoThrottle + - Throttling + * - ``AUTOTHROTTLE_ENABLED = True`` + - No equivalent; throttling is always active. + * - ``AUTOTHROTTLE_START_DELAY`` + - :setting:`DOWNLOAD_DELAY` + * - ``AUTOTHROTTLE_MAX_DELAY`` + - :setting:`BACKOFF_MAX_DELAY` + * - ``AUTOTHROTTLE_TARGET_CONCURRENCY`` + - :ref:`rampup ` (``"rampup": True``) + * - ``AUTOTHROTTLE_DEBUG = True`` + - :setting:`THROTTLING_DEBUG` ``= True`` + * - ``autothrottle_dont_adjust_delay`` (request meta) + - :reqmeta:`throttling_dont_track` (request meta) + +AutoThrottle adjusted the delay of each download slot based on response +latency. The new system does not measure latency; instead, it reacts to +explicit rate-limit signals (:setting:`BACKOFF_HTTP_CODES`, +:setting:`BACKOFF_EXCEPTIONS`, :ref:`Retry-After / RateLimit-Reset +`) and, with :ref:`rampup `, probes for the +fastest rate a scope tolerates. If you specifically need latency-based control, +implement a custom :ref:`throttling scope manager +`. + .. _throttling-api: @@ -761,330 +1028,10 @@ API .. autoclass:: scrapy.throttling.ThrottlingScopeManager +.. autoclass:: scrapy.throttling.ThrottlingScopeConfig + +.. autoclass:: scrapy.throttling.BackoffConfig + .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope .. autofunction:: scrapy.throttling.update_scope_backoff - - - - - - -.. - When a throttling scope is configured with a **concurrency higher than 1**, - backoff is handled separately per slot. If at some point all - slots reach the maximum backoff delay, a “concurrency backoff” - starts, controlled by the following setting: - - - .. setting:: BACKOFF_CONCURRENCY_DECREASE_FACTOR - - :setting:`BACKOFF_CONCURRENCY_FACTOR` (default: ``0.5``) - - The factor by which the concurrency is decreased during concurrency - backoff. - - .. _scope-backoff: - - Backoff settings can be overridden per throttling scope using - :setting:`THROTTLING_SCOPES`: - - .. code-block:: python - - { - "example.com": { - "backoff": { - "http_codes": [429, 503], - "exceptions": ["builtins.IOError"], - "delay_factor": 1.2, - "max_delay": 180.0, - "min_delay": 5.0, - "jitter": [0.01, 0.33], - "concurrency_factor": 0.8, - } - }, - } - -.. - TODO: Continue from here - -.. - TODO: Support backoff in THROTTLING_SCOPES having a cls key with a class - or its import path as a string, and arbitraty kwargs to pass to its - __init__ method, to implement a custom backoff strategy. - - -.. - TODO: Make sure that the API is flexible enough to support scrapy-zyte-api - use cases. - -.. - TODO: Try to figure out the implementation for the entire feature, and how - that may affect the user-facing API. - -.. - TODO: Think about: - - How backoff state is handled. - - Implement a setting that allows customizing the backoff window, which - should be 60.0 seconds by default, and should be overridable per scope. - - How backoff could be fine-tuned to deal with scenarios where it is - ping-poning between 2 states, and it should be possible to go for an - in-between state that allows maximizing throughtput while still - minimizing backoff feedback (backoff response or exception), aiming for a - single backoff feedback per backoff window. - -.. - TODO: Figure out the interaction APIs between components when we have: - - The scheduler using the throttling manager to decide whether a request - can be sent or not, and to sort requests. - -.. - TODO: Review all related issues and PRs, including those about multiple - slots, about per-request delays, and about politeness, and make sure every - scenario is covered here. - -.. - TODO: See if there is any info from the old autothrottle docs worth - keeping: - - .. _topics-autothrottle: - - ====================== - AutoThrottle extension - ====================== - - This is an extension for automatically throttling crawling speed based on load - of both the Scrapy server and the website you are crawling. - - Design goals - ============ - - 1. be nicer to sites instead of using default download delay of zero - 2. automatically adjust Scrapy to the optimum crawling speed, so the user - doesn't have to tune the download delays to find the optimum one. - The user only needs to specify the maximum concurrent requests - it allows, and the extension does the rest. - - .. _autothrottle-algorithm: - - How it works - ============ - - Scrapy allows defining the concurrency and delay of different download slots, - e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are - assigned to slots based on their URL domain, although it is possible to - customize the download slot of any request. - - The AutoThrottle extension adjusts the delay of each download slot dynamically, - to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent - requests on average to each remote website. - - It uses download latency to compute the delays. The main idea is the - following: if a server needs ``latency`` seconds to respond, a client - should send a request each ``latency/N`` seconds to have ``N`` requests - processed in parallel. - - Instead of adjusting the delays one can just set a small fixed - download delay and impose hard limits on concurrency using - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or - :setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar - effect, but there are some important differences: - - * because the download delay is small there will be occasional bursts - of requests; - * often non-200 (error) responses can be returned faster than regular - responses, so with a small download delay and a hard concurrency limit - crawler will be sending requests to server faster when server starts to - return errors. But this is an opposite of what crawler should do - in case - of errors it makes more sense to slow down: these errors may be caused by - the high request rate. - - AutoThrottle doesn't have these issues. - - Throttling algorithm - ==================== - - AutoThrottle algorithm adjusts download delays based on the following rules: - - 1. spiders always start with a download delay of - :setting:`AUTOTHROTTLE_START_DELAY`; - 2. when a response is received, the target download delay is calculated as - ``latency / N`` where ``latency`` is a latency of the response, - and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`. - 3. download delay for next requests is set to the average of previous - download delay and the target download delay; - 4. latencies of non-200 responses are not allowed to decrease the delay; - 5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater - than :setting:`AUTOTHROTTLE_MAX_DELAY` - - .. note:: The AutoThrottle extension honours the standard Scrapy settings for - concurrency and delay. This means that it will respect - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and - never set a download delay lower than :setting:`DOWNLOAD_DELAY`. - - .. _download-latency: - - In Scrapy, the download latency is measured as the time elapsed between - establishing the TCP connection and receiving the HTTP headers. - - Note that these latencies are very hard to measure accurately in a cooperative - multitasking environment because Scrapy may be busy processing a spider - callback, for example, and unable to attend downloads. However, these latencies - should still give a reasonable estimate of how busy Scrapy (and ultimately, the - server) is, and this extension builds on that premise. - - .. reqmeta:: autothrottle_dont_adjust_delay - - Prevent specific requests from triggering slot delay adjustments - ================================================================ - - AutoThrottle adjusts the delay of download slots based on the latencies of - responses that belong to that download slot. The only exceptions are non-200 - responses, which are only taken into account to increase that delay, but - ignored if they would decrease that delay. - - You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to - ``True`` in any request to prevent its response latency from impacting the - delay of its download slot: - - .. code-block:: python - - from scrapy import Request - - Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True}) - - Note, however, that AutoThrottle still determines the starting delay of every - download slot by setting the ``download_delay`` attribute on the running - spider. If you want AutoThrottle not to impact a download slot at all, in - addition to setting this meta key in all requests that use that download slot, - you might want to set a custom value for the ``delay`` attribute of that - download slot, e.g. using :setting:`DOWNLOAD_SLOTS`. - - Settings - ======== - - The settings used to control the AutoThrottle extension are: - - * :setting:`AUTOTHROTTLE_ENABLED` - * :setting:`AUTOTHROTTLE_START_DELAY` - * :setting:`AUTOTHROTTLE_MAX_DELAY` - * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` - * :setting:`AUTOTHROTTLE_DEBUG` - * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - * :setting:`CONCURRENT_REQUESTS_PER_IP` - * :setting:`DOWNLOAD_DELAY` - - For more information see :ref:`autothrottle-algorithm`. - - .. setting:: AUTOTHROTTLE_ENABLED - - AUTOTHROTTLE_ENABLED - ~~~~~~~~~~~~~~~~~~~~ - - Default: ``False`` - - Enables the AutoThrottle extension. - - .. setting:: AUTOTHROTTLE_START_DELAY - - AUTOTHROTTLE_START_DELAY - ~~~~~~~~~~~~~~~~~~~~~~~~ - - Default: ``5.0`` - - The initial download delay (in seconds). - - .. setting:: AUTOTHROTTLE_MAX_DELAY - - AUTOTHROTTLE_MAX_DELAY - ~~~~~~~~~~~~~~~~~~~~~~ - - Default: ``60.0`` - - The maximum download delay (in seconds) to be set in case of high latencies. - - .. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY - - AUTOTHROTTLE_TARGET_CONCURRENCY - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - Default: ``1.0`` - - Average number of requests Scrapy should be sending in parallel to remote - websites. It must be higher than ``0.0``. - - By default, AutoThrottle adjusts the delay to send a single - concurrent request to each of the remote websites. Set this option to - a higher value (e.g. ``2.0``) to increase the throughput and the load on remote - servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value - (e.g. ``0.5``) makes the crawler more conservative and polite. - - Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected - when AutoThrottle extension is enabled. This means that if - ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or - :setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number - of concurrent requests. - - At every given time point Scrapy can be sending more or less concurrent - requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested - value the crawler tries to approach, not a hard limit. - - .. setting:: AUTOTHROTTLE_DEBUG - - AUTOTHROTTLE_DEBUG - ~~~~~~~~~~~~~~~~~~ - - Default: ``False`` - - Enable AutoThrottle debug mode which will display stats on every response - received, so you can see how the throttling parameters are being adjusted in - real time. - -.. - TODO: Figure out how to properly deprecate AutoThrottle settings, API and - keep its presence in the list of enabled extensions without that triggering - a deprecation warning. - -.. - TODO: Avoid so much code duplication in add_scope and update_scope_backoff. - -.. - TODO: Describe in detail the backoff algorithm, with steps, etc., and also - covering how it tries to maintain a sweet spot avoiding - -.. - TODO: Define typed dicts for all dicts supported by settings, e.g. for - THROTTLING_SCOPES. - -.. - TODO: Provide an example implementation of a custom throttling scope - manager based on the current implementation of scrapy-zyte-api retrying - policy. - -.. - TODO: If get_scopes reports expected quotas, treat those as actual - consumptions at run time, and then handle the actual consumptions reported - in get_response_backoff by reporting the difference. - - e.g. if a request is expected to consume 2.0 quota, lower the available - quota in the current window by 2.0. If the response then reports that 2.5 - was actually consumed, then report 0.5 as the difference, to be also - substracted from the current window. - -.. - TODO: Implement some kind of system to free scope-related memory if a given - scope is not used for a while? - -.. - TODO: Think about tracking effective concurrency and delays, and reporting - to users when their custom settings are not effective for a significant - period of time. - -.. - TODO: Provide a complete list of keys that THROTTLING_SCOPES supports. - -.. - TODO: Update news.rst with all the changes, but only once everything else - has been written. Make sure to include the new REDIRECT_MAX_DELAY setting. From 7fdee93e4202a3ee72390719ab86e281d07b0bfd Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 25 Jun 2026 13:38:55 +0200 Subject: [PATCH 09/55] WIP --- docs/conf.py | 1 - docs/topics/signals.rst | 19 ++++++ scrapy/core/downloader/__init__.py | 2 + scrapy/core/engine.py | 72 ++++++++++++++++++-- scrapy/downloadermiddlewares/robotstxt.py | 12 +++- scrapy/extensions/throttle.py | 1 + scrapy/robotstxt.py | 18 +++++ scrapy/settings/default_settings.py | 30 ++++++++ scrapy/signals.py | 1 + tests/test_downloadermiddleware_robotstxt.py | 18 +++++ 10 files changed, 166 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 44de57421..2fc7afe8c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -145,7 +145,6 @@ coverage_ignore_pyobjects = [ # -- Options for the autodoc extension ---------------------------------------- autodoc_type_aliases = { "BackoffData": "BackoffData", - "GetScopesMethod": "GetScopesMethod", "RequestScopes": "RequestScopes", } diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d13733623..845d7ab47 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -502,6 +502,25 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a ``robots.txt`` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed ``robots.txt`` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the ``robots.txt`` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..a9778d1d7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,6 +86,8 @@ def _get_concurrency_delay( delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay + if settings.get("DOWNLOAD_DELAY_PER_SLOT") is not None: + delay = settings.getfloat("DOWNLOAD_DELAY_PER_SLOT") if hasattr(spider, "max_concurrent_requests"): # pragma: no cover warn_on_deprecated_spider_attribute( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1033e874f..9096cff5a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -129,6 +129,13 @@ class ExecutionEngine: self._start_request_processing_awaitable: ( asyncio.Future[None] | Deferred[None] | None ) = None + # Requests currently held by the throttling manager, waiting for their + # scopes to allow them through to the downloader. + self._throttling_waiting: set[Request] = set() + self._delayed_requests_warn_threshold: int = self.settings.getint( + "DELAYED_REQUESTS_WARN_THRESHOLD" + ) + self._delayed_requests_warned: bool = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -351,6 +358,35 @@ class ExecutionEngine: or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() + or self._throttling_needs_backout() + ) + + def _throttling_needs_backout(self) -> bool: + """Apply backpressure while requests are held by the throttling manager. + + Requests waiting on throttling do not occupy a downloader concurrency + slot, so without this check the engine would keep draining the + scheduler into pending throttling waits. Count them together with the + in-flight requests against the global concurrency limit. + """ + if not self._throttling_waiting: + return False + return ( + len(self._throttling_waiting) + len(self.downloader.active) + >= self.downloader.total_concurrency + ) + + def _maybe_warn_delayed_requests(self) -> None: + if self._delayed_requests_warned: + return + if len(self._throttling_waiting) < self._delayed_requests_warn_threshold: + return + self._delayed_requests_warned = True + logger.warning( + f"There are {len(self._throttling_waiting)} requests held back by throttling. This may " + "indicate a throttling configuration that is too aggressive or a " + "site that is rate-limiting heavily. See DELAYED_REQUESTS_WARN_THRESHOLD.", + extra={"spider": self.spider}, ) def _remove_request(self, _: Any, request: Request) -> None: @@ -426,6 +462,8 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False + if self._throttling_waiting: # requests held by the throttling manager + return False if self._start is not None: # not all start requests are handled return False return not self._slot.scheduler.has_pending_requests() @@ -481,6 +519,21 @@ class ExecutionEngine: return await self.download_async(response_or_request) return response_or_request + @inlineCallbacks + def _acquire_throttling( + self, request: Request + ) -> Generator[Deferred[Any], Any, None]: + """Wait at the throttling gate before *request* is sent, tracking it as + held meanwhile.""" + self._throttling_waiting.add(request) + self._maybe_warn_delayed_requests() + throttler = self.crawler.throttler + assert throttler is not None + try: + yield deferred_from_coro(throttler.acquire(request)) + finally: + self._throttling_waiting.discard(request) + @inlineCallbacks def _download( self, request: Request @@ -489,12 +542,19 @@ class ExecutionEngine: assert self.spider is not None self._slot.add_request(request) + throttler = self.crawler.throttler + assert throttler is not None try: - result: Response | Request - if self._downloader_fetch_needs_spider: - result = yield self.downloader.fetch(request, self.spider) - else: - result = yield self.downloader.fetch(request) + yield self._acquire_throttling(request) + try: + result: Response | Request + if self._downloader_fetch_needs_spider: + result = yield self.downloader.fetch(request, self.spider) + else: + result = yield self.downloader.fetch(request) + except Exception as exc: + yield deferred_from_coro(throttler.process_exception(request, exc)) + raise if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -502,6 +562,7 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request + yield deferred_from_coro(throttler.process_response(result)) logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: logger.log( @@ -515,6 +576,7 @@ class ExecutionEngine: ) return result finally: + throttler.release(request) self._slot.nextcall.schedule() def open_spider( diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 89fa6ff67..42e58cbc9 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -31,6 +31,7 @@ class AutoThrottle: "backoff settings instead: " "https://docs.scrapy.org/en/latest/topics/throttling.html", ScrapyDeprecationWarning, + stacklevel=2, ) self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..dc3ba9426 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,16 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + :param user_agent: User agent + :type user_agent: str or bytes + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +95,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +135,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 049921ba7..709d55038 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -235,6 +235,19 @@ AWS_SESSION_TOKEN = None AWS_USE_SSL = None AWS_VERIFY = None +BACKOFF_DELAY_FACTOR = 2.0 +BACKOFF_EXCEPTIONS = [ + "scrapy.exceptions.DownloadFailedError", + "scrapy.exceptions.DownloadTimeoutError", + "scrapy.exceptions.ResponseDataLossError", +] +BACKOFF_HTTP_CODES = [429, 502, 503, 504, 520, 521, 522, 523, 524] +BACKOFF_JITTER = 0.1 +BACKOFF_MAX_DELAY = 300.0 +BACKOFF_MIN_DELAY = 1.0 +BACKOFF_WINDOW = 60.0 + + BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 @@ -267,6 +280,8 @@ DEFAULT_REQUEST_HEADERS = { "Accept-Language": "en", } +DELAYED_REQUESTS_WARN_THRESHOLD = 500 + DEPTH_LIMIT = 0 DEPTH_PRIORITY = 0 DEPTH_STATS_VERBOSE = False @@ -279,6 +294,7 @@ DNS_TIMEOUT = 60 DOWNLOAD_BIND_ADDRESS = None DOWNLOAD_DELAY = 0 +DOWNLOAD_DELAY_PER_SLOT = None DOWNLOAD_FAIL_ON_DATALOSS = True @@ -488,6 +504,8 @@ PERIODIC_LOG_DELTA = None PERIODIC_LOG_STATS = None PERIODIC_LOG_TIMING_ENABLED = False +RAMPUP_BACKOFF_TARGET = 1 + RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 @@ -566,6 +584,8 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] +TARGET_RPM = None + TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_PORT = [6023, 6073] @@ -574,9 +594,19 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager" +THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager" +THROTTLING_SCOPES = {} +THROTTLING_WINDOW = 60.0 +THROTTLING_ROBOTSTXT_OBEY = True +THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLING_SCOPE_MAX_IDLE = 3600.0 +THROTTLING_DEBUG = False + TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" TWISTED_REACTOR_ENABLED = True + TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" URLLENGTH_LIMIT = 2083 diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 082fc743e..061a551e5 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -74,6 +76,22 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + assert "crawler" not in calls[0] + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler()) From ec18ddc23974486ac0ebc285c2c38e311ef07aa3 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 25 Jun 2026 16:01:47 +0200 Subject: [PATCH 10/55] Initial implementation draft --- docs/topics/throttling.rst | 12 +- scrapy/settings/default_settings.py | 2 - scrapy/throttling.py | 890 ++++++++++++++++++++++++---- scrapy/utils/asyncio.py | 91 ++- tests/test_throttling.py | 567 ++++++++++++++++++ 5 files changed, 1439 insertions(+), 123 deletions(-) create mode 100644 tests/test_throttling.py diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 852b9f8e9..436d1e336 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -45,6 +45,10 @@ The main throttling :ref:`settings ` are: Even if you have multiple slots, requests to the same domain cannot be sent more frequently than this delay. + To target a specific number of requests per minute (RPM) *per domain*, set + this to ``60 / RPM``. For example, ``DOWNLOAD_DELAY = 1.0`` for 60 RPM, or + ``DOWNLOAD_DELAY = 2.0`` for 30 RPM. + - .. setting:: DOWNLOAD_DELAY_PER_SLOT :setting:`DOWNLOAD_DELAY_PER_SLOT` (default: ``None``) @@ -945,14 +949,6 @@ Additional settings If ``True``, ``0.5`` (i.e. ±50%) is used as the randomization factor. If ``False``, no randomization is applied. -- .. setting:: TARGET_RPM - - :setting:`TARGET_RPM` (default: ``None``) - - Target number of requests per minute *per domain*. It has no effect on its - own; it is read by :class:`~scrapy.addons.throttling.TargetRPMAddon`, which - must be enabled explicitly. - - .. setting:: THROTTLING_DEBUG :setting:`THROTTLING_DEBUG` (default: ``False``) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 709d55038..f19894658 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -584,8 +584,6 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] -TARGET_RPM = None - TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_PORT = [6023, 6073] diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 1f5168cd5..9b70ec23f 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1,29 +1,38 @@ from __future__ import annotations +import contextlib import datetime as dt -from collections.abc import Awaitable, Iterable -from datetime import UTC +import logging +import random +import time +from collections.abc import Awaitable, Callable, Iterable from email.utils import parsedate_to_datetime from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Protocol, TypedDict, Union +from typing import TYPE_CHECKING, Any, Protocol, TypedDict, TypeVar, cast from weakref import WeakKeyDictionary +from twisted.internet.defer import Deferred from typing_extensions import NotRequired, Self -from scrapy.http import Request, Response +from scrapy import signals +from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.http import Request, Response + + +logger = logging.getLogger(__name__) def _parse_retry_after(response: Response) -> float | None: - value = response.headers.get("Retry-After") - if not value: + raw = response.headers.get("Retry-After") + if not raw: return None try: - value = value.decode("utf-8").strip() + value = raw.decode("utf-8").strip() except UnicodeDecodeError: return None if value.isdigit(): @@ -33,18 +42,18 @@ def _parse_retry_after(response: Response) -> float | None: except (TypeError, ValueError, OverflowError): return None if date.tzinfo is None: - date = date.replace(tzinfo=UTC) - now = dt.datetime.now(UTC) + date = date.replace(tzinfo=dt.timezone.utc) + now = dt.datetime.now(dt.timezone.utc) seconds_to_wait = (date - now).total_seconds() return max(0, int(seconds_to_wait)) or None def _parse_ratelimit_reset(response: Response) -> float | None: - value = response.headers.get("RateLimit-Reset") - if not value: + raw = response.headers.get("RateLimit-Reset") + if not raw: return None try: - value = value.decode("utf-8").strip() + value = raw.decode("utf-8").strip() except UnicodeDecodeError: return None try: @@ -59,9 +68,50 @@ class BackoffScopeData(TypedDict): remaining: NotRequired[float] +class BackoffConfig(TypedDict, total=False): + """Per-scope override of the backoff settings. + + Used as the value of the ``"backoff"`` key of :class:`ThrottlingScopeConfig` + entries. Any key left out falls back to the corresponding global + ``BACKOFF_*`` setting. + """ + + http_codes: list[int] + exceptions: list[str] + delay_factor: float + max_delay: float + min_delay: float + jitter: float | list[float] + + +class ThrottlingScopeConfig(TypedDict, total=False): + """Accepted keys of :setting:`THROTTLING_SCOPES` entries. + + Every key is optional; missing keys fall back to the matching global + setting (e.g. ``delay`` falls back to :setting:`DOWNLOAD_DELAY`). + """ + + concurrency: int + + min_concurrency: int + """Floor. Never drop below this during backoff/rampup.""" + + delay: float + jitter: float | list[float] + quota: float + window: float + rampup: bool + + manager: str | type + """Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for + this scope.""" + + backoff: BackoffConfig + + ScopeID = str -BackoffData = Union[None, ScopeID, Iterable[ScopeID], dict[ScopeID, BackoffScopeData]] -RequestScopes = Union[None, ScopeID, Iterable[ScopeID], dict[ScopeID, float | None]] +BackoffData = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, BackoffScopeData] +RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None] def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: @@ -74,6 +124,61 @@ def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: return iter(scopes) +def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | None]]: + """Iterate over *scopes* as ``(scope_id, value)`` pairs. + + For dict scopes the value is the expected :ref:`throttling quota + ` consumption; for every other form the value is + ``None``. + """ + if scopes is None: + return + if isinstance(scopes, str): + yield scopes, None + return + if isinstance(scopes, dict): + yield from scopes.items() + return + for scope in scopes: + yield scope, None + + +def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]: + """Normalize *collection* (``None``, str, iterable or dict) into a dict + mapping scope names to values produced by *default*.""" + if isinstance(collection, dict): + return collection + if collection is None: + return {} + if isinstance(collection, str): + return {collection: default()} + if isinstance(collection, Iterable): + return {scope: default() for scope in collection} + raise TypeError( + f"Invalid type ({type(collection)}) of scopes value " + f"{collection!r}. Expected None, str, Iterable or dict." + ) + + +def _add_bare_scope(collection: Any, scope: ScopeID, empty: Any) -> Any: + """Add *scope* to *collection* without any associated value, keeping the + most compact representation possible.""" + if collection is None: + return scope + if isinstance(collection, str): + return collection if collection == scope else {collection, scope} + if isinstance(collection, dict): + if scope not in collection: + collection[scope] = empty + return collection + if isinstance(collection, Iterable): + return set(collection) | {scope} if scope not in collection else collection + raise TypeError( + f"Invalid type ({type(collection)}) of scopes value " + f"{collection!r}. Expected None, str, Iterable or dict." + ) + + def add_scope( scopes: RequestScopes, scope: ScopeID, @@ -86,38 +191,12 @@ def add_scope( :meth:`~ThrottlingManagerProtocol.get_scopes`, e.g. in :class:`ThrottlingManager` subclasses. """ - if value is not None: - if not isinstance(scopes, dict): - if scopes is None: - scopes = {} - elif isinstance(scopes, str): - scopes = {scopes: None} - elif isinstance(scopes, Iterable): - scopes = {s: None for s in scopes} - else: - raise TypeError( - f"Invalid type ({type(scopes)}) of scopes value " - f"{scopes!r}. Expected None, str, Iterable or dict." - ) - if scope in scopes and not isinstance(scopes[scope], dict): - raise TypeError(f"Scope {scope!r} has a non-dict value in {scopes!r}") - scopes[scope] = value - elif scopes is None: - scopes = scope - elif isinstance(scopes, str): - if scopes != scope: - scopes = {scopes, scope} - elif isinstance(scopes, dict): - if scope not in scopes: - scopes[scope] = None - elif isinstance(scopes, Iterable): - if scope not in scopes: - scopes = set(scopes) | {scope} - else: - raise TypeError( - f"Invalid type ({type(scopes)}) of scopes value " - f"{scopes!r}. Expected None, str, Iterable or dict." - ) + if value is None: + return cast("RequestScopes", _add_bare_scope(scopes, scope, None)) + scopes = _to_scope_dict(scopes, lambda: None) + if scope in scopes and not isinstance(scopes[scope], dict): + raise TypeError(f"Scope {scope!r} has a non-dict value in {scopes!r}") + scopes[scope] = value return scopes @@ -129,7 +208,7 @@ def update_scope_backoff( delay: float | None = None, consumed: float | None = None, ) -> BackoffData: - """Add *scope* to *backoff* or update its existing entry the given + """Add *scope* to *backoff* or update its existing entry with the given parameters. This is a utility function to help extending the output of @@ -138,45 +217,16 @@ def update_scope_backoff( :meth:`~ThrottlingManagerProtocol.get_exception_backoff`, e.g. in :class:`ThrottlingManager` subclasses. """ - has_params = delay is not None or consumed is not None - if has_params: - if not isinstance(backoff, dict): - if backoff is None: - backoff = {} - elif isinstance(backoff, str): - backoff = {backoff: {}} - elif isinstance(backoff, Iterable): - backoff = {s: {} for s in backoff} - else: - raise TypeError( - f"Invalid type ({type(backoff)}) of scopes value " - f"{backoff!r}. Expected None, str, Iterable or dict." - ) - if scope in backoff: - if not isinstance(backoff[scope], dict): - raise TypeError(f"Scope {scope!r} has a non-dict value in {backoff!r}") - else: - backoff[scope] = {} - if delay is not None: - backoff[scope]["delay"] = delay - if consumed is not None: - backoff[scope]["consumed"] = consumed - elif backoff is None: - backoff = scope - elif isinstance(backoff, str): - if backoff != scope: - backoff = {backoff, scope} - elif isinstance(backoff, dict): - if scope not in backoff: - backoff[scope] = {} - elif isinstance(backoff, Iterable): - if scope not in backoff: - backoff = set(backoff) | {scope} - else: - raise TypeError( - f"Invalid type ({type(backoff)}) of scopes value " - f"{backoff!r}. Expected None, str, Iterable or dict." - ) + if delay is None and consumed is None: + return cast("BackoffData", _add_bare_scope(backoff, scope, {})) + backoff = _to_scope_dict(backoff, dict) + entry = backoff.setdefault(scope, {}) + if not isinstance(entry, dict): + raise TypeError(f"Scope {scope!r} has a non-dict value in {backoff!r}") + if delay is not None: + entry["delay"] = delay + if consumed is not None: + entry["consumed"] = consumed return backoff @@ -251,13 +301,35 @@ class ThrottlingManagerProtocol(Protocol): It supports the same return values as :meth:`get_initial_backoff`. """ + async def acquire(self, request: Request) -> None: + """Block until *request* is allowed to be sent by all of its scopes. -GetScopesMethod = Callable[ - [ThrottlingManagerProtocol, Request], Awaitable[RequestScopes] -] + This is the throttling gate that the engine awaits before releasing a + request to the downloader. + """ + + def release(self, request: Request) -> None: + """Release the concurrency slots that :meth:`acquire` reserved for + *request*. + + The engine calls this once *request* has finished downloading (whether + it succeeded, failed or returned a new request), so that scopes that + enforce a concurrency limit can let other requests through. + """ + + async def process_response(self, response: Response) -> None: + """Update the throttling state based on *response*.""" + + async def process_exception(self, request: Request, exception: Exception) -> None: + """Update the throttling state based on a download *exception*.""" -def scope_cache(f: GetScopesMethod) -> GetScopesMethod: +_GetScopesMethod = TypeVar( + "_GetScopesMethod", bound=Callable[..., Awaitable[RequestScopes]] +) + + +def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: """Decorator to cache the result of :meth:`~ThrottlingManagerProtocol.get_scopes` calls. @@ -280,17 +352,17 @@ def scope_cache(f: GetScopesMethod) -> GetScopesMethod: async def get_scopes(self, request): return urlparse_cached(request).netloc """ - cache = WeakKeyDictionary() + cache: WeakKeyDictionary[Request, RequestScopes] = WeakKeyDictionary() @wraps(f) - async def wrapper(self, request: Request): + async def wrapper(self: Any, request: Request) -> RequestScopes: if request in cache: return cache[request] scopes = await f(self, request) cache[request] = scopes return scopes - return wrapper + return wrapper # type: ignore[return-value] class ThrottlingManager: @@ -306,27 +378,57 @@ class ThrottlingManager: def __init__(self, crawler: Crawler) -> None: self.crawler = crawler - self.throttler = crawler.throttler - self.backoff_http_codes = set(crawler.settings.getlist("BACKOFF_HTTP_CODES")) - self.backoff_exceptions = tuple( + self._backoff_http_codes = { + int(code) for code in crawler.settings.getlist("BACKOFF_HTTP_CODES") + } + self._backoff_exceptions = tuple( load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS") ) + self._debug = crawler.settings.getbool("THROTTLING_DEBUG") + self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") + self._robotstxt_obey = crawler.settings.getbool( + "ROBOTSTXT_OBEY" + ) and crawler.settings.getbool("THROTTLING_ROBOTSTXT_OBEY") + self._robotstxt_max_delay = crawler.settings.getfloat( + "THROTTLING_ROBOTSTXT_MAX_DELAY" + ) + self._default_useragent: str = crawler.settings["USER_AGENT"] + self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] + if self._robotstxt_obey: + crawler.signals.connect( + self._on_robots_parsed, signal=signals.robots_parsed + ) + self._default_scope_manager_cls = load_object( + crawler.settings["THROTTLING_SCOPE_MANAGER"] + ) + self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict( + "THROTTLING_SCOPES" + ) + self._scope_managers: dict[ScopeID, ThrottlingScopeManagerProtocol] = {} + self._last_eviction: float | None = None + # Concurrency slots reserved by acquire(), to be released once the + # request finishes downloading. + self._reserved: WeakKeyDictionary[ + Request, list[tuple[ThrottlingScopeManagerProtocol, float | None]] + ] = WeakKeyDictionary() @scope_cache - async def get_scopes( - self: ThrottlingManagerProtocol, request: Request - ) -> RequestScopes: + async def get_scopes(self, request: Request) -> RequestScopes: + scopes = request.meta.get("throttling_scopes") + if scopes is not None: + return cast("RequestScopes", scopes) return urlparse_cached(request).netloc async def get_initial_backoff(self) -> BackoffData: return None async def get_response_backoff(self, response: Response) -> BackoffData: - if response.status not in self.backoff_http_codes: - return None assert response.request is not None - assert self.throttler is not None - scopes = await self.throttler.get_scopes(response.request) + if response.request.meta.get("throttling_dont_track"): + return None + if response.status not in self._backoff_http_codes: + return None + scopes = await self.get_scopes(response.request) if delay := self.get_response_delay(response): scopes = {scope: {"delay": delay} for scope in iter_scopes(scopes)} return scopes @@ -347,11 +449,207 @@ class ThrottlingManager: async def get_exception_backoff( self, request: Request, exception: Exception ) -> BackoffData: - if isinstance(exception, self.backoff_exceptions): - assert self.throttler is not None - return await self.throttler.get_scopes(request) + if request.meta.get("throttling_dont_track"): + return None + if isinstance(exception, self._backoff_exceptions): + return await self.get_scopes(request) return None + # -- Scope-state coordination (called from the request lifecycle) -------- + + def _get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol: + manager = self._scope_managers.get(scope_id) + if manager is None: + config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {})) + config.setdefault("id", scope_id) + manager_cls = ( + load_object(config["manager"]) + if "manager" in config + else self._default_scope_manager_cls + ) + manager = build_from_crawler(manager_cls, self.crawler, config) + self._scope_managers[scope_id] = manager + return manager + + async def acquire(self, request: Request) -> None: + now = time.monotonic() + self._maybe_evict(now) + await self._apply_request_delay(request) + scope_values = list(iter_scope_values(await self.get_scopes(request))) + if not scope_values: + return + managers = [ + (self._get_scope_manager(scope_id), value) + for scope_id, value in scope_values + ] + while True: + wait = max( + [0.0, *(manager.can_send(amount=value) for manager, value in managers)] + ) + if wait > 0: + if self._debug: + logger.debug( + f"Throttling {request} for {wait:.2f}s " + f"(scopes: {[scope_id for scope_id, _ in scope_values]})" + ) + await sleep(wait) + continue + # All time-based gates (delay, backoff, quota) are open; the only + # remaining reason to wait is a full concurrency slot. + blocked = [ + manager for manager, _ in managers if manager.concurrency_blocked() + ] + if not blocked: + for manager, value in managers: + manager.record_sent(amount=value) + self._reserved[request] = managers + return + if self._debug: + logger.debug( + f"Throttling {request} until a concurrency slot frees up " + f"(scopes: {[scope_id for scope_id, _ in scope_values]})" + ) + await self._wait_for_slot(blocked) + + def release(self, request: Request) -> None: + managers = self._reserved.pop(request, None) + if not managers: + return + for manager, _ in managers: + manager.record_done() + + async def _wait_for_slot(self, managers: list[Any]) -> None: + """Block until any of *managers* frees a concurrency slot. + + Each manager hands out an event Deferred that fires when a slot is freed + (via :meth:`ThrottlingScopeManager.record_done`) or the limit is raised + (via :meth:`ThrottlingScopeManager.set_concurrency`). A long safety timer + bounds the wait in case no slot is ever freed (it always should be, via + :meth:`release`). + """ + pairs = [(manager, manager.slot_event()) for manager in managers] + events = [event for _, event in pairs] + _, pending = await wait_for_first(events, timeout=_SLOT_WAIT_TIMEOUT) + for manager, event in pairs: + if event in pending: + manager.discard_slot_event(event) + + async def _apply_request_delay(self, request: Request) -> None: + """Honor the :reqmeta:`throttling_delay` meta key by holding *request* + for the requested number of seconds the first time it is processed.""" + delay = request.meta.get("throttling_delay") + if not delay or request.meta.get("_throttling_delayed"): + return + request.meta["_throttling_delayed"] = True + if self._debug: + logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") + await sleep(float(delay)) + + async def process_response(self, response: Response) -> None: + data = await self.get_response_backoff(response) + self._apply_backoff(data) + + async def process_exception(self, request: Request, exception: Exception) -> None: + data = await self.get_exception_backoff(request, exception) + self._apply_backoff(data) + + def _apply_backoff(self, data: BackoffData) -> None: + if data is None: + return + if isinstance(data, dict): + items: Iterable[tuple[ScopeID, Any]] = data.items() + else: + items = ((scope_id, None) for scope_id in iter_scopes(data)) + for scope_id, entry in items: + manager = self._get_scope_manager(scope_id) + delay = consumed = remaining = None + if isinstance(entry, dict): + delay = entry.get("delay") + consumed = entry.get("consumed") + remaining = entry.get("remaining") + # A dict entry that only reports quota usage reconciles the quota + # without counting as a backoff trigger. + quota_only = ( + isinstance(entry, dict) + and delay is None + and (consumed is not None or remaining is not None) + ) + if consumed is not None or remaining is not None: + manager.reconcile_quota(consumed=consumed, remaining=remaining) + if quota_only: + continue + if self._debug: + logger.debug(f"Backoff for scope {scope_id} (delay: {delay})") + manager.record_backoff(delay=delay) + + def _on_robots_parsed(self, robotparser: Any, request: Request) -> None: + """Honor a robots.txt ``Crawl-delay`` on the :signal:`robots_parsed` + signal. + + It reads the ``Crawl-delay`` directive for the configured user agent from + the parsed robots.txt and, if present, applies it to the scope of the + host that *request* targets via :meth:`apply_robots_crawl_delay`. + """ + if not self._robotstxt_obey: + return + useragent: str | bytes = self._robotstxt_useragent or self._default_useragent + try: + delay = robotparser.crawl_delay(useragent) + except Exception: # pragma: no cover - backend-specific failures + return + if delay: + self.apply_robots_crawl_delay(urlparse_cached(request).netloc, delay) + + def apply_robots_crawl_delay(self, scope_id: ScopeID, delay: float) -> None: + """Honor a robots.txt ``Crawl-delay`` directive of *delay* seconds for + *scope_id* by setting its delay (capped at + :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY`) and its concurrency to ``1``. + + Called from the :signal:`robots_parsed` signal handler when + :setting:`THROTTLING_ROBOTSTXT_OBEY` is enabled. An explicit + :setting:`THROTTLING_SCOPES` configuration for the scope is respected, but + a warning is logged about the discrepancy unless its ``ignore_robots_txt`` + key is ``True``. + """ + if not self._robotstxt_obey: + return + capped = min(delay, self._robotstxt_max_delay) + config = self._scopes_config.get(scope_id, {}) + if config.get("ignore_robots_txt"): + return + conflicts = [] + if config.get("delay") is not None and float(config["delay"]) < capped: + conflicts.append(f"delay={config['delay']!r} < Crawl-delay {capped}") + if config.get("concurrency") is not None and int(config["concurrency"]) > 1: + conflicts.append(f"concurrency={config['concurrency']!r} > 1") + if conflicts: + logger.warning( + f"Throttling scope {scope_id!r} is configured with {' and '.join(conflicts)}, " + f"which is more aggressive than its robots.txt Crawl-delay of " + f"{capped}s. The configured values take precedence; set " + "'ignore_robots_txt': True in its THROTTLING_SCOPES entry to " + "silence this warning." + ) + return + if self._debug: + logger.debug(f"robots.txt Crawl-delay for scope {scope_id}: {capped}s") + manager = self._get_scope_manager(scope_id) + manager.set_base_delay(capped) + manager.set_concurrency(1) + + def _maybe_evict(self, now: float) -> None: + if self._max_idle <= 0: + return + if ( + self._last_eviction is not None + and now - self._last_eviction < self._max_idle / 2 + ): + return + self._last_eviction = now + for scope_id in list(self._scope_managers): + if self._scope_managers[scope_id].is_idle(now, self._max_idle): + del self._scope_managers[scope_id] + class ThrottlingScopeManagerProtocol(Protocol): """A protocol for :setting:`THROTTLING_SCOPE_MANAGER` :ref:`components @@ -364,7 +662,7 @@ class ThrottlingScopeManagerProtocol(Protocol): { "id": "example.com", - "concurrency": 1.0, + "concurrency": 1, "delay": 1.0, "jitter": 0.5, "quota": 1000.0, @@ -376,7 +674,6 @@ class ThrottlingScopeManagerProtocol(Protocol): "max_delay": 180.0, "min_delay": 5.0, "jitter": [0.01, 0.33], - "concurrency_factor": 0.8, }, "rampup": { "backoff_target": 1, @@ -384,6 +681,7 @@ class ThrottlingScopeManagerProtocol(Protocol): "min_delay": 0.05, }, } + """ @classmethod @@ -393,6 +691,378 @@ class ThrottlingScopeManagerProtocol(Protocol): def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: pass + def can_send(self, now: float | None = None, amount: float | None = None) -> float: + """Return the number of seconds to wait before a request for this scope + may be sent, or ``0`` if it may be sent right away. + + *amount* is the expected :ref:`throttling quota ` + consumption of the request, if any. + """ + + def record_sent( + self, now: float | None = None, amount: float | None = None + ) -> None: + """Record that a request for this scope has just been sent, consuming + *amount* of its :ref:`throttling quota ` if given.""" + + def record_done(self, now: float | None = None) -> None: + """Record that a previously :meth:`record_sent` request has finished + downloading, freeing its concurrency slot.""" + + def record_backoff( + self, delay: float | None = None, now: float | None = None + ) -> None: + """Apply a backoff to this scope. + + *delay*, when given, is a hard minimum delay in seconds (e.g. from a + ``Retry-After`` header). When omitted, an exponential backoff step is + applied instead. + """ + + def reconcile_quota( + self, + consumed: float | None = None, + remaining: float | None = None, + now: float | None = None, + ) -> None: + """Reconcile the :ref:`throttling quota ` of this + scope with the actual *consumed* amount (or the *remaining* amount) + reported for a request, correcting the estimate used by + :meth:`record_sent`.""" + + def set_base_delay(self, delay: float) -> None: + """Raise the base (non-backoff) delay of this scope to *delay* seconds. + + It never lowers the configured base delay; it is used to honor external + hints such as a robots.txt ``Crawl-delay`` directive. + """ + + def set_concurrency(self, concurrency: int) -> None: + """Set the maximum number of concurrent requests allowed for this + scope.""" + + def concurrency_blocked(self) -> bool: + """Return whether this scope is at its concurrency limit. + + :class:`ThrottlingManager` calls this (after all time-based gates in + :meth:`can_send` are open) to decide whether to wait for a freed slot. + Return ``False`` when no concurrency limit is enforced. + """ + + def slot_event(self) -> Deferred: + """Return a :class:`~twisted.internet.defer.Deferred` that fires when + a concurrency slot next becomes available (e.g. when + :meth:`record_done` is called or the limit is raised via + :meth:`set_concurrency`).""" + + def discard_slot_event(self, event: Deferred) -> None: + """Cancel a pending slot event returned by :meth:`slot_event`. + + Called by :class:`ThrottlingManager` when the wait ends without the + event firing (e.g. another scope's slot opened first). + """ + + def is_idle(self, now: float, max_idle: float) -> bool: + """Return whether this scope can be evicted from memory. + + A scope is idle when it has not been used for *max_idle* seconds and is + not currently in an active (future) backoff. + """ + + +# Safety timeout for acquire() while it waits, event-driven, for a concurrency +# slot to free up: a slot_event normally fires first (from record_done() or +# set_concurrency()); this only guards against a request that never reaches +# release() so the wait can never hang forever. +_SLOT_WAIT_TIMEOUT = 1.0 + class ThrottlingScopeManager: - """The default :setting:`THROTTLING_SCOPE_MANAGER` class.""" + """The default :setting:`THROTTLING_SCOPE_MANAGER` class. + + It implements a per-scope state machine covering delay, exponential + :ref:`backoff `, :ref:`rampup `, concurrency and + :ref:`quotas `: + + - A base :setting:`DOWNLOAD_DELAY`-style delay (``0`` by default, taken + from the scope ``"delay"`` config) is enforced between consecutive + requests for the scope. + + - On a backoff trigger (a :setting:`BACKOFF_HTTP_CODES` response or a + :setting:`BACKOFF_EXCEPTIONS` exception) the delay grows exponentially + by :setting:`BACKOFF_DELAY_FACTOR`, bounded by :setting:`BACKOFF_MIN_DELAY` + and :setting:`BACKOFF_MAX_DELAY`, with :setting:`BACKOFF_JITTER` applied. + A ``Retry-After`` / ``RateLimit-Reset`` delay is honored as a hard + minimum (capped at :setting:`BACKOFF_MAX_DELAY`). + + - After :setting:`BACKOFF_WINDOW` seconds without a new trigger, the delay + recovers one step at a time back towards the base delay. + + - When the scope is configured with a ``"concurrency"`` limit (or with + ``"rampup"``), no more than that many requests are allowed in flight at + once, never dropping below the ``"min_concurrency"`` floor. + + - When the scope sets ``"rampup": True``, throughput is increased every + :setting:`BACKOFF_WINDOW` that stays under :setting:`RAMPUP_BACKOFF_TARGET` + backoff triggers, first by lowering the delay and then by raising the + concurrency limit. + + - When the scope is configured with a ``"quota"``, no more than that much + quota is consumed per ``"window"`` (default: :setting:`THROTTLING_WINDOW`). + """ + + @classmethod + def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self: + return cls(crawler, config) + + def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: + settings = crawler.settings + backoff: dict[str, Any] = config.get("backoff", {}) + self._id: ScopeID = config.get("id", "") + self._base_delay: float = float(config.get("delay", 0.0)) + self._randomize: bool = bool( + config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")) + ) + self._delay_factor: float = float( + backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR")) + ) + self._max_delay: float = float( + backoff.get("max_delay", settings.getfloat("BACKOFF_MAX_DELAY")) + ) + self._min_delay: float = float( + backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY")) + ) + self._jitter: float | list[float] = backoff.get( + "jitter", settings.getfloat("BACKOFF_JITTER") + ) + self._window: float = settings.getfloat("BACKOFF_WINDOW") + self._min_concurrency: int = int(config.get("min_concurrency", 1)) + + # Rampup. + rampup = config.get("rampup") + self._rampup_enabled: bool = bool(rampup) + rampup_config: dict[str, Any] = rampup if isinstance(rampup, dict) else {} + self._rampup_target: tuple[float, float] = self._parse_target( + rampup_config.get("backoff_target", settings.get("RAMPUP_BACKOFF_TARGET")) + ) + self._rampup_delay_factor: float = float(rampup_config.get("delay_factor", 0.5)) + self._rampup_min_delay: float = float(rampup_config.get("min_delay", 0.0)) + + # Concurrency. ``None`` means no scope-level limit (the downloader slots + # enforce concurrency instead); a limit is only set when configured + # explicitly or implied by rampup. + configured_concurrency = config.get("concurrency") + if configured_concurrency is not None: + self._concurrency: int | None = int(configured_concurrency) + elif self._rampup_enabled: + self._concurrency = self._min_concurrency + else: + self._concurrency = None + + # Quota. + quota = config.get("quota") + self._quota: float | None = None if quota is None else float(quota) + self._quota_window: float = float( + config.get("window", settings.getfloat("THROTTLING_WINDOW")) + ) + + # State. + self._delay: float = self._base_delay + self._backoff_level: int = 0 + self._next_allowed_time: float | None = None + self._in_backoff_until: float | None = None + self._last_backoff_time: float | None = None + self._last_seen: float | None = None + self._active: int = 0 + self._slot_waiters: list[Deferred[None]] = [] + self._consumed: float = 0.0 + self._quota_window_start: float | None = None + self._rampup_window_start: float | None = None + self._rampup_backoffs: int = 0 + + @staticmethod + def _now(now: float | None) -> float: + return time.monotonic() if now is None else now + + @staticmethod + def _parse_target(value: Any) -> tuple[float, float]: + if isinstance(value, (list, tuple)): + return float(value[0]), float(value[1]) + return float(value), float(value) + + def _apply_jitter(self, value: float) -> float: + if isinstance(self._jitter, (list, tuple)): + low, high = self._jitter[0], self._jitter[1] + return value * (1 + random.uniform(low, high)) # noqa: S311 + if not self._jitter: + return value + return value * random.uniform(1 - self._jitter, 1 + self._jitter) # noqa: S311 + + def _effective_delay(self) -> float: + if self._backoff_level == 0 and self._randomize and self._delay > 0: + return random.uniform(0.5 * self._delay, 1.5 * self._delay) # noqa: S311 + return self._delay + + def _recover(self, now: float) -> None: + if self._backoff_level == 0 or self._last_backoff_time is None: + return + while self._backoff_level > 0 and now - self._last_backoff_time >= self._window: + self._backoff_level -= 1 + self._last_backoff_time += self._window + if self._backoff_level == 0: + self._delay = self._base_delay + self._in_backoff_until = None + self._last_backoff_time = None + break + self._delay = max(self._base_delay, self._delay / self._delay_factor) + + def _maybe_rampup(self, now: float) -> None: + """Increase throughput once per :setting:`BACKOFF_WINDOW` that stays + under :setting:`RAMPUP_BACKOFF_TARGET` backoff triggers.""" + if not self._rampup_enabled: + return + if self._rampup_window_start is None: + self._rampup_window_start = now + return + while now - self._rampup_window_start >= self._window: + self._rampup_window_start += self._window + if self._rampup_backoffs < self._rampup_target[0]: + self._rampup_step() + self._rampup_backoffs = 0 + + def _rampup_step(self) -> None: + # Backoff in progress: let it recover before probing again. + if self._backoff_level > 0: + return + if self._delay > self._rampup_min_delay: + self._delay = max( + self._rampup_min_delay, self._delay * self._rampup_delay_factor + ) + self._base_delay = min(self._base_delay, self._delay) + elif self._concurrency is not None: + self._concurrency += 1 + + def _maybe_reset_quota(self, now: float) -> None: + if self._quota is None: + return + if self._quota_window_start is None: + self._quota_window_start = now + return + while now - self._quota_window_start >= self._quota_window: + self._quota_window_start += self._quota_window + self._consumed = 0.0 + + def can_send(self, now: float | None = None, amount: float | None = None) -> float: + now = self._now(now) + self._recover(now) + self._maybe_rampup(now) + self._maybe_reset_quota(now) + waits = [0.0] + if self._in_backoff_until is not None: + waits.append(self._in_backoff_until - now) + if self._next_allowed_time is not None: + waits.append(self._next_allowed_time - now) + if self._quota is not None: + need = 0.0 if amount is None else float(amount) + # Block until the window resets only if some quota is already spent; + # a single oversized request is always allowed through. + if self._consumed > 0 and self._consumed + need > self._quota: + start = self._quota_window_start or now + waits.append(start + self._quota_window - now) + # Concurrency is enforced separately, via concurrency_blocked() and + # slot_event(), so acquire() can wait for a freed slot without polling. + return max(waits) + + def record_sent( + self, now: float | None = None, amount: float | None = None + ) -> None: + now = self._now(now) + self._last_seen = now + if self._in_backoff_until is not None and now >= self._in_backoff_until: + self._in_backoff_until = None + self._next_allowed_time = now + self._effective_delay() + self._active += 1 + if self._quota is not None and amount is not None: + self._maybe_reset_quota(now) + self._consumed += float(amount) + + def record_done(self, now: float | None = None) -> None: + if self._active > 0: + self._active -= 1 + self._fire_slot_waiters() + + def concurrency_blocked(self) -> bool: + return self._concurrency is not None and self._active >= self._concurrency + + def slot_event(self) -> Deferred[None]: + """Return a Deferred that fires when a concurrency slot next frees up + (via :meth:`record_done`) or the limit is raised (via + :meth:`set_concurrency`).""" + event: Deferred[None] = Deferred() + self._slot_waiters.append(event) + return event + + def discard_slot_event(self, event: Deferred[None]) -> None: + with contextlib.suppress(ValueError): + self._slot_waiters.remove(event) + + def _fire_slot_waiters(self) -> None: + waiters, self._slot_waiters = self._slot_waiters, [] + for event in waiters: + if not event.called: + event.callback(None) + + def record_backoff( + self, delay: float | None = None, now: float | None = None + ) -> None: + now = self._now(now) + self._last_seen = now + self._last_backoff_time = now + self._backoff_level += 1 + self._rampup_backoffs += 1 + if delay is not None: + hard = min(float(delay), self._max_delay) + self._in_backoff_until = now + hard + self._delay = min(max(self._delay, hard, self._min_delay), self._max_delay) + else: + grown = ( + self._delay * self._delay_factor if self._delay > 0 else self._min_delay + ) + grown = max(self._min_delay, grown) + grown = min(grown, self._max_delay) + self._delay = min(self._apply_jitter(grown), self._max_delay) + self._next_allowed_time = now + self._delay + + def reconcile_quota( + self, + consumed: float | None = None, + remaining: float | None = None, + now: float | None = None, + ) -> None: + if self._quota is None: + return + self._maybe_reset_quota(self._now(now)) + if remaining is not None: + self._consumed = max(0.0, self._quota - float(remaining)) + elif consumed is not None: + self._consumed = max(0.0, self._consumed + float(consumed)) + + def set_base_delay(self, delay: float) -> None: + if delay <= self._base_delay: + return + self._base_delay = delay + if self._backoff_level == 0: + self._delay = delay + + def set_concurrency(self, concurrency: int) -> None: + self._concurrency = max(self._min_concurrency, int(concurrency)) + self._fire_slot_waiters() + + def is_idle(self, now: float, max_idle: float) -> bool: + if self._in_backoff_until is not None and self._in_backoff_until > now: + return False + if self._active > 0: + return False + if self._last_seen is None: + return True + return (now - self._last_seen) > max_idle diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..71aecde9c 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -5,11 +5,11 @@ from __future__ import annotations import asyncio import logging import time -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable +from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Sequence from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar -from twisted.internet.defer import Deferred -from twisted.internet.task import LoopingCall +from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread from scrapy.utils.asyncgen import as_async_generator @@ -311,3 +311,88 @@ async def run_in_thread( from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 return await maybe_deferred_to_future(deferToThread(func, *args, **kwargs)) + + +async def sleep(seconds: float) -> None: + """Sleep for *seconds*, working in asyncio-reactor, non-asyncio-reactor, + and reactorless modes. + + Uses :func:`asyncio.sleep` when asyncio is available, and + :func:`~twisted.internet.task.deferLater` otherwise. + + .. versionadded:: VERSION + """ + if is_asyncio_available(): + await asyncio.sleep(seconds) + else: + from twisted.internet import reactor + + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + await maybe_deferred_to_future(deferLater(reactor, seconds, lambda: None)) + + +async def wait_for_first( + deferreds: Sequence[Deferred[Any]], + *, + timeout: float | None = None, +) -> tuple[set[Deferred[Any]], set[Deferred[Any]]]: + """Wait for the first of *deferreds* to fire, or until *timeout* seconds pass. + + Returns ``(done, pending)`` — two sets partitioning the input + :class:`~twisted.internet.defer.Deferred` objects — mirroring the API of + :func:`asyncio.wait`. + + Unfired deferreds in the ``pending`` set are neither cancelled nor + otherwise modified; the caller is responsible for any cleanup. + + Returns ``(set(), set())`` immediately when *deferreds* is empty. + + Works transparently in asyncio-reactor, non-asyncio-reactor, and + reactorless modes. + + .. versionadded:: VERSION + """ + if not deferreds: + return set(), set() + + if is_asyncio_available(): + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + future_to_deferred = {maybe_deferred_to_future(d): d for d in deferreds} + done_futures, pending_futures = await asyncio.wait( + list(future_to_deferred), + timeout=timeout, + return_when=asyncio.FIRST_COMPLETED, + ) + return ( + {future_to_deferred[f] for f in done_futures}, + {future_to_deferred[f] for f in pending_futures}, + ) + + from twisted.internet import reactor + + # circular import + from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 + + timeout_deferred = ( + deferLater(reactor, timeout, lambda: None) if timeout is not None else None + ) + waiter = DeferredList( + [*deferreds, *([] if timeout_deferred is None else [timeout_deferred])], + fireOnOneCallback=True, + fireOnOneErrback=True, + consumeErrors=True, + ) + try: + await maybe_deferred_to_future(waiter) + finally: + if timeout_deferred is not None and not timeout_deferred.called: + timeout_deferred.cancel() + + return ( + {d for d in deferreds if d.called}, + {d for d in deferreds if not d.called}, + ) diff --git a/tests/test_throttling.py b/tests/test_throttling.py new file mode 100644 index 000000000..0e0269c95 --- /dev/null +++ b/tests/test_throttling.py @@ -0,0 +1,567 @@ +from __future__ import annotations + +import logging + +import pytest + +from scrapy import signals +from scrapy.exceptions import DownloadTimeoutError +from scrapy.http import Request, Response +from scrapy.throttling import ThrottlingManager, ThrottlingScopeManager +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.test import get_crawler +from tests.spiders import SimpleSpider +from tests.utils.decorators import coroutine_test + + +def _manager(settings=None): + crawler = get_crawler(settings_dict=settings) + return ThrottlingManager.from_crawler(crawler) + + +def _scope_manager(settings=None, config=None): + crawler = get_crawler(settings_dict=settings) + return ThrottlingScopeManager.from_crawler(crawler, config or {"id": "example.com"}) + + +class _FakeRobotParser: + """A minimal robots.txt parser stub for :signal:`robots_parsed` tests. + + *delay* is returned by :meth:`crawl_delay`, unless it is an exception, in + which case it is raised to emulate a backend-specific failure. + """ + + def __init__(self, delay): + self._delay = delay + + def crawl_delay(self, useragent): + if isinstance(self._delay, Exception): + raise self._delay + return self._delay + + +def _response(status=200, headers=None, url="http://example.com", meta=None): + request = Request(url, meta=meta or {}) + return Response(url, status=status, headers=headers or {}, request=request) + + +class TestThrottlingManager: + @coroutine_test + async def test_get_scopes_returns_netloc(self): + manager = _manager() + assert ( + await manager.get_scopes(Request("http://example.com/a")) == "example.com" + ) + + @coroutine_test + async def test_get_scopes_cached(self): + manager = _manager() + request = Request("http://example.com/a") + first = await manager.get_scopes(request) + # A second call returns the cached value (same object identity for dicts, + # equal value for strings). + assert await manager.get_scopes(request) == first + + @coroutine_test + async def test_get_scopes_meta_string(self): + manager = _manager() + request = Request("http://example.com/a", meta={"throttling_scopes": "api"}) + assert await manager.get_scopes(request) == "api" + + @coroutine_test + async def test_get_scopes_meta_dict(self): + manager = _manager() + request = Request( + "http://example.com/a", meta={"throttling_scopes": {"api": 2.0}} + ) + assert await manager.get_scopes(request) == {"api": 2.0} + + @coroutine_test + async def test_get_initial_backoff_none(self): + manager = _manager() + assert await manager.get_initial_backoff() is None + + def test_scope_manager_class_in_config(self): + manager = _manager( + {"THROTTLING_SCOPES": {"example.com": {"manager": ThrottlingScopeManager}}} + ) + scope = manager._get_scope_manager("example.com") + assert isinstance(scope, ThrottlingScopeManager) + + def test_release_frees_concurrency(self): + manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) + scope = manager._get_scope_manager("example.com") + request = Request("http://example.com") + scope.record_sent(now=0.0) + manager._reserved[request] = [(scope, None)] + assert scope.concurrency_blocked() is True + manager.release(request) + assert scope.concurrency_blocked() is False + # Releasing again is a no-op. + manager.release(request) + assert scope.concurrency_blocked() is False + + @coroutine_test + async def test_acquire_waits_for_freed_slot(self): + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + + manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) + r1 = Request("http://example.com/1") + r2 = Request("http://example.com/2") + # Drive acquire() the way the engine does, so it runs as a real task that + # can await the slot event under the asyncio reactor. + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r1))) + scope = manager._get_scope_manager("example.com") + assert scope.concurrency_blocked() is True + assert scope.concurrency_blocked() is True + # acquire(r2) must block until r1 frees the slot; release it on the next + # event loop tick so the event-driven wait wakes up. + call_later(0, manager.release, r1) + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r2))) + assert scope.concurrency_blocked() is True + assert r2 in manager._reserved + + def test_apply_backoff_reconciles_quota_without_backoff(self): + manager = _manager({"THROTTLING_SCOPES": {"cost": {"quota": 100.0}}}) + scope = manager._get_scope_manager("cost") + manager._apply_backoff({"cost": {"consumed": 5.0}}) + # Quota was reconciled but no backoff step was applied. + assert scope._consumed == pytest.approx(5.0) + assert scope._backoff_level == 0 + + def test_apply_backoff_delay_and_consumed(self): + manager = _manager({"THROTTLING_SCOPES": {"cost": {"quota": 100.0}}}) + scope = manager._get_scope_manager("cost") + manager._apply_backoff({"cost": {"delay": 5.0, "consumed": 2.0}}) + assert scope._consumed == pytest.approx(2.0) + assert scope._backoff_level == 1 + + @coroutine_test + async def test_response_backoff_non_backoff_code(self): + manager = _manager() + assert await manager.get_response_backoff(_response(status=200)) is None + + @coroutine_test + async def test_response_backoff_429_without_header(self): + manager = _manager() + assert ( + await manager.get_response_backoff(_response(status=429)) == "example.com" + ) + + @pytest.mark.parametrize( + ("header", "expected_delay"), + [ + ({"Retry-After": "7"}, 7.0), + ({"RateLimit-Reset": "12"}, 12.0), + ], + ids=["retry-after", "ratelimit-reset"], + ) + @coroutine_test + async def test_response_backoff_delay_header(self, header, expected_delay): + manager = _manager() + data = await manager.get_response_backoff(_response(status=429, headers=header)) + assert data == {"example.com": {"delay": expected_delay}} + + @coroutine_test + async def test_response_backoff_retry_after_http_date(self): + manager = _manager() + # A date far in the past yields no positive delay. + data = await manager.get_response_backoff( + _response( + status=503, + headers={"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}, + ) + ) + assert data == "example.com" + + @coroutine_test + async def test_response_backoff_max_of_both_headers(self): + manager = _manager() + data = await manager.get_response_backoff( + _response( + status=429, + headers={"Retry-After": "5", "RateLimit-Reset": "9"}, + ) + ) + assert data == {"example.com": {"delay": 9.0}} + + @coroutine_test + async def test_response_backoff_dont_track(self): + manager = _manager() + response = _response(status=429, meta={"throttling_dont_track": True}) + assert await manager.get_response_backoff(response) is None + + @pytest.mark.parametrize( + ("exception", "expected"), + [ + (DownloadTimeoutError(), "example.com"), + (ValueError(), None), + ], + ids=["tracked", "untracked"], + ) + @coroutine_test + async def test_exception_backoff(self, exception, expected): + manager = _manager() + request = Request("http://example.com") + assert await manager.get_exception_backoff(request, exception) == expected + + @coroutine_test + async def test_exception_backoff_dont_track(self): + manager = _manager() + request = Request("http://example.com", meta={"throttling_dont_track": True}) + assert ( + await manager.get_exception_backoff(request, DownloadTimeoutError()) is None + ) + + @pytest.mark.parametrize( + ("settings", "parser_delay", "expected_base_delay"), + [ + ({"ROBOTSTXT_OBEY": True, "RANDOMIZE_DOWNLOAD_DELAY": False}, 3.0, 3.0), + ({"ROBOTSTXT_OBEY": True, "THROTTLING_ROBOTSTXT_OBEY": False}, 3.0, 0.0), + ({"ROBOTSTXT_OBEY": True}, None, 0.0), + ({"ROBOTSTXT_OBEY": True}, ValueError(), 0.0), + ], + ids=["applies-delay", "obey-disabled", "no-delay", "backend-error"], + ) + def test_robots_parsed_signal(self, settings, parser_delay, expected_base_delay): + manager = _manager(settings) + manager.crawler.signals.send_catch_log( + signal=signals.robots_parsed, + robotparser=_FakeRobotParser(parser_delay), + request=Request("http://example.com/page"), + ) + scope = manager._get_scope_manager("example.com") + assert scope._base_delay == expected_base_delay + + def test_apply_robots_crawl_delay(self): + manager = _manager({"ROBOTSTXT_OBEY": True, "RANDOMIZE_DOWNLOAD_DELAY": False}) + manager.apply_robots_crawl_delay("example.com", 3.0) + scope = manager._get_scope_manager("example.com") + assert scope._base_delay == 3.0 + assert scope.can_send(now=0) == 0 # nothing sent yet + scope.record_sent(now=0) + assert scope.can_send(now=0) == pytest.approx(3.0) + + def test_apply_robots_crawl_delay_capped(self): + manager = _manager( + {"ROBOTSTXT_OBEY": True, "THROTTLING_ROBOTSTXT_MAX_DELAY": 2.0} + ) + manager.apply_robots_crawl_delay("example.com", 30.0) + assert manager._get_scope_manager("example.com")._base_delay == 2.0 + + def test_apply_robots_crawl_delay_disabled(self): + manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLING_ROBOTSTXT_OBEY": False}) + manager.apply_robots_crawl_delay("example.com", 3.0) + assert manager._get_scope_manager("example.com")._base_delay == 0.0 + + def test_apply_robots_crawl_delay_sets_concurrency(self): + manager = _manager({"ROBOTSTXT_OBEY": True}) + manager.apply_robots_crawl_delay("example.com", 3.0) + assert manager._get_scope_manager("example.com")._concurrency == 1 + + def test_apply_robots_crawl_delay_warns_on_conflict(self, caplog): + manager = _manager( + { + "ROBOTSTXT_OBEY": True, + "THROTTLING_SCOPES": {"example.com": {"concurrency": 8}}, + } + ) + with caplog.at_level(logging.WARNING, logger="scrapy.throttling"): + manager.apply_robots_crawl_delay("example.com", 3.0) + assert "Crawl-delay" in caplog.text + # The configured value takes precedence (crawl-delay not applied). + assert manager._get_scope_manager("example.com")._base_delay == 0.0 + + def test_apply_robots_crawl_delay_ignored(self, caplog): + manager = _manager( + { + "ROBOTSTXT_OBEY": True, + "THROTTLING_SCOPES": { + "example.com": {"concurrency": 8, "ignore_robots_txt": True} + }, + } + ) + with caplog.at_level(logging.WARNING, logger="scrapy.throttling"): + manager.apply_robots_crawl_delay("example.com", 3.0) + # No warning is logged and the crawl-delay is not applied. + assert "Crawl-delay" not in caplog.text + assert manager._get_scope_manager("example.com")._base_delay == 0.0 + + def test_scope_eviction(self): + manager = _manager({"THROTTLING_SCOPE_MAX_IDLE": 100.0}) + scope = manager._get_scope_manager("example.com") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # Not idle yet. + manager._last_eviction = None + manager._maybe_evict(now=50.0) + assert "example.com" in manager._scope_managers + # Idle past the threshold. + manager._last_eviction = None + manager._maybe_evict(now=201.0) + assert "example.com" not in manager._scope_managers + + def test_scope_eviction_skips_active_backoff(self): + manager = _manager( + {"THROTTLING_SCOPE_MAX_IDLE": 100.0, "BACKOFF_MAX_DELAY": 100_000.0} + ) + scope = manager._get_scope_manager("example.com") + scope.record_backoff(delay=10_000.0, now=0.0) + manager._last_eviction = None + manager._maybe_evict(now=5_000.0) + # Still in backoff (in_backoff_until far in the future), so not evicted + # even though it has been idle for longer than THROTTLING_SCOPE_MAX_IDLE. + assert "example.com" in manager._scope_managers + + +class TestThrottlingScopeManager: + def test_no_delay_by_default(self): + scope = _scope_manager() + scope.record_sent(now=0.0) + assert scope.can_send(now=0.0) == 0 + + def test_base_delay_enforced(self): + scope = _scope_manager( + {"RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x", "delay": 2.0} + ) + scope.record_sent(now=10.0) + assert scope.can_send(now=10.0) == pytest.approx(2.0) + assert scope.can_send(now=11.0) == pytest.approx(1.0) + assert scope.can_send(now=12.0) == 0 + + def test_exponential_backoff(self): + scope = _scope_manager( + { + "BACKOFF_MIN_DELAY": 1.0, + "BACKOFF_DELAY_FACTOR": 2.0, + "BACKOFF_JITTER": 0, + }, + ) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(1.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(2.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(4.0) + + def test_backoff_cap(self): + scope = _scope_manager( + { + "BACKOFF_MIN_DELAY": 1.0, + "BACKOFF_DELAY_FACTOR": 10.0, + "BACKOFF_MAX_DELAY": 5.0, + "BACKOFF_JITTER": 0, + }, + ) + for _ in range(5): + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(5.0) + + @pytest.mark.parametrize( + ("max_delay", "backoff_delay", "expected"), + [ + (100.0, 20.0, 20.0), + (10.0, 999.0, 10.0), + ], + ids=["within-cap", "capped"], + ) + def test_retry_after_delay(self, max_delay, backoff_delay, expected): + scope = _scope_manager({"BACKOFF_MAX_DELAY": max_delay}) + scope.record_backoff(delay=backoff_delay, now=0.0) + assert scope.can_send(now=0.0) == pytest.approx(expected) + + def test_recovery_after_window(self): + scope = _scope_manager( + { + "BACKOFF_MIN_DELAY": 1.0, + "BACKOFF_DELAY_FACTOR": 2.0, + "BACKOFF_WINDOW": 60.0, + "BACKOFF_JITTER": 0, + }, + ) + scope.record_backoff(now=0.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(2.0) + # One window passes with no new backoff -> one step down. + scope.can_send(now=60.0) + assert scope._backoff_level == 1 + assert scope._delay == pytest.approx(1.0) + # Another window -> back to base (0). + scope.can_send(now=120.0) + assert scope._backoff_level == 0 + assert scope._delay == pytest.approx(0.0) + + def test_per_scope_backoff_override(self): + scope = _scope_manager( + {"BACKOFF_MIN_DELAY": 1.0, "BACKOFF_DELAY_FACTOR": 2.0}, + { + "id": "x", + "backoff": {"min_delay": 5.0, "delay_factor": 3.0, "jitter": 0}, + }, + ) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(5.0) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(15.0) + + def test_set_base_delay_raises_only(self): + scope = _scope_manager( + {"RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x", "delay": 5.0} + ) + scope.set_base_delay(2.0) # lower -> ignored + assert scope._base_delay == 5.0 + scope.set_base_delay(8.0) # higher -> applied + assert scope._base_delay == 8.0 + assert scope._delay == 8.0 + + def test_min_delay_first_step(self): + scope = _scope_manager( + {"BACKOFF_MIN_DELAY": 3.0, "BACKOFF_DELAY_FACTOR": 2.0, "BACKOFF_JITTER": 0} + ) + scope.record_backoff(now=0.0) + assert scope._delay == pytest.approx(3.0) + + def test_no_scope_concurrency_limit_by_default(self): + scope = _scope_manager() + assert scope._concurrency is None + for _ in range(100): + scope.record_sent(now=0.0) + assert scope.can_send(now=0.0) == 0 + assert scope.concurrency_blocked() is False + + def test_concurrency_limit(self): + scope = _scope_manager(config={"id": "x", "concurrency": 2}) + scope.record_sent(now=0.0) + # Concurrency is enforced via concurrency_blocked(), not can_send(). + assert scope.can_send(now=0.0) == 0 + assert scope.concurrency_blocked() is False + scope.record_sent(now=0.0) + # Two in flight, limit reached -> blocked. + assert scope.can_send(now=0.0) == 0 + assert scope.concurrency_blocked() is True + scope.record_done(now=0.0) + assert scope.concurrency_blocked() is False + + def test_record_done_fires_slot_event(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + scope.record_sent(now=0.0) + event = scope.slot_event() + assert not event.called + scope.record_done(now=0.0) + assert event.called + + def test_set_concurrency_fires_slot_event(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + scope.record_sent(now=0.0) + event = scope.slot_event() + assert not event.called + scope.set_concurrency(5) + assert event.called + + def test_discard_slot_event(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + event = scope.slot_event() + scope.discard_slot_event(event) + scope.discard_slot_event(event) # idempotent + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + assert not event.called + + def test_set_concurrency_respects_min(self): + scope = _scope_manager(config={"id": "x", "min_concurrency": 3}) + scope.set_concurrency(1) + assert scope._concurrency == 3 + scope.set_concurrency(5) + assert scope._concurrency == 5 + + def test_quota_blocks_when_exhausted(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0, "window": 60.0}) + scope.record_sent(now=0.0, amount=6.0) + assert scope.can_send(now=0.0, amount=3.0) == 0 # 9 <= 10 + scope.record_sent(now=0.0, amount=3.0) + # 9 spent; a 3.0 request would exceed the quota -> wait for the window. + assert scope.can_send(now=0.0, amount=3.0) == pytest.approx(60.0) + # The window resets and quota is available again. + assert scope.can_send(now=60.0, amount=3.0) == 0 + + def test_quota_allows_oversized_request(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + # A single request larger than the whole quota is still allowed. + assert scope.can_send(now=0.0, amount=999.0) == 0 + + def test_quota_reconcile_consumed_delta(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + scope.record_sent(now=0.0, amount=2.0) + assert scope._consumed == pytest.approx(2.0) + # The response reports it actually consumed 0.5 more than estimated. + scope.reconcile_quota(consumed=0.5, now=0.0) + assert scope._consumed == pytest.approx(2.5) + + def test_quota_reconcile_remaining(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + scope.record_sent(now=0.0, amount=2.0) + scope.reconcile_quota(remaining=3.0, now=0.0) + assert scope._consumed == pytest.approx(7.0) + + def test_rampup_lowers_delay_when_quiet(self): + scope = _scope_manager( + {"BACKOFF_WINDOW": 10.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, + { + "id": "x", + "delay": 4.0, + "rampup": {"delay_factor": 0.5, "min_delay": 0.5}, + }, + ) + scope.can_send(now=0.0) # start the rampup window + # A quiet window (no backoff) lowers the delay. + scope.can_send(now=10.0) + assert scope._delay == pytest.approx(2.0) + scope.can_send(now=20.0) + assert scope._delay == pytest.approx(1.0) + + def test_rampup_raises_concurrency_at_min_delay(self): + scope = _scope_manager( + {"BACKOFF_WINDOW": 10.0}, + {"id": "x", "delay": 0.0, "rampup": True, "min_concurrency": 1}, + ) + assert scope._concurrency == 1 + scope.can_send(now=0.0) + scope.can_send(now=10.0) + assert scope._concurrency == 2 + + def test_rampup_holds_when_target_met(self): + scope = _scope_manager( + {"BACKOFF_WINDOW": 10.0, "RAMPUP_BACKOFF_TARGET": 1}, + {"id": "x", "delay": 0.0, "rampup": True, "min_concurrency": 1}, + ) + scope.can_send(now=0.0) + scope.record_backoff(now=1.0) # one trigger == target -> hold, do not probe + scope.can_send(now=10.0) + assert scope._concurrency == 1 + + +class TestThrottlingIntegration: + @coroutine_test + async def test_backoff_recorded_on_429(self, mockserver): + crawler = get_crawler(SimpleSpider, {"RETRY_ENABLED": False}) + await crawler.crawl_async( + mockserver.url("/status?n=429"), mockserver=mockserver + ) + throttler = crawler.throttler + assert throttler is not None + managers = throttler._scope_managers + assert managers, "no throttling scope was created" + assert any(manager._backoff_level >= 1 for manager in managers.values()) + + @coroutine_test + async def test_no_backoff_on_200(self, mockserver): + crawler = get_crawler(SimpleSpider) + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) + throttler = crawler.throttler + assert throttler is not None + assert all( + manager._backoff_level == 0 + for manager in throttler._scope_managers.values() + ) From 8190b5b703362b1f70c7d2022e02ccd7a5601a55 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 10:44:31 +0200 Subject: [PATCH 11/55] Add a throttling-aware scheduler --- docs/topics/throttling.rst | 62 +++++++++++-- scrapy/core/engine.py | 86 +++++++++++++++++- scrapy/core/scheduler.py | 108 ++++++++++++++++++++++ scrapy/pqueues.py | 178 +++++++++++++++++++++++++++++++++++++ scrapy/throttling.py | 121 ++++++++++++++++++++++++- tests/test_pqueues.py | 112 ++++++++++++++++++++++- tests/test_scheduler.py | 174 +++++++++++++++++++++++++++++++++++- tests/test_throttling.py | 85 ++++++++++++++++++ 8 files changed, 911 insertions(+), 15 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 436d1e336..cc8519990 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -459,8 +459,7 @@ path as a string): THROTTLING_MANAGER = "myproject.throttling.MyThrottlingManager" - -.. _multi-throttling-scopes: +.. _multiple-throttling-scopes: Handling of multiple throttling scopes -------------------------------------- @@ -468,7 +467,6 @@ Handling of multiple throttling scopes When a request has multiple throttling scopes, it is not sent until all of its throttling scopes allow it. - .. _throttling-quotas: Throttling quotas @@ -505,7 +503,6 @@ Everything else being equal, :class:`~scrapy.pqueues.ScrapyPriorityQueue` will prioritize requests that consume a higher portion of the available throttling quota, to minimize the risk of those requests getting stuck. - .. _custom-throttling-scope-managers: Customizing throttling scope managers @@ -603,6 +600,28 @@ exponential backoff, similar to a fixed-window rate limiter: def is_idle(self, now, max_idle): return self.active == 0 +.. _throttling-aware-scheduler: + +Throttling-aware scheduling +=========================== + +By default, throttling is enforced at the engine, where a request waiting on +its :ref:`throttling scopes ` holds a concurrency slot. In a +crawl that mixes heavily-throttled scopes with unthrottled ones, this can let +throttled requests starve unthrottled ones that could be sent right away +(**head-of-line blocking**; Scrapy logs a warning, see +:setting:`DELAYED_REQUESTS_WARN_THRESHOLD`). + +:class:`~scrapy.core.scheduler.ThrottlingAwareScheduler` avoids this. To enable +it: + +.. code-block:: python + :caption: ``settings.py`` + + SCHEDULER = "scrapy.core.scheduler.ThrottlingAwareScheduler" + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ThrottlingAwarePriorityQueue" + +.. autoclass:: scrapy.core.scheduler.ThrottlingAwareScheduler .. _throttling-examples: @@ -822,8 +841,8 @@ window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas - Implement a :ref:`throttling manager ` that: - - Sets a ``cost`` throttling scope on each request to some estimation based - e.g. on request URL parameters: + - Sets a ``cost`` throttling scope on each request to some estimation + based e.g. on request URL parameters: .. code-block:: python @@ -871,6 +890,35 @@ window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas This will allow you to spend up to 100.0 units of cost per time window (default: 60 seconds) before throttling kicks in. +.. _throttling-per-ip: + +Per-IP concurrency limiting +--------------------------- + +A concurrency limit keyed by IP is just a throttling scope whose id is the +request's IP, with a ``concurrency`` limit. A request then carries two scopes, +its domain and its IP, and is only sent when **both** allow it (see +:ref:`multiple-throttling-scopes`). + +- Implement a :ref:`throttling manager ` that adds + the request's IP as a second scope: + + .. code-block:: python + + import socket + + from scrapy.throttling import ThrottlingManager, add_scope, scope_cache + from scrapy.utils.asyncio import run_in_thread + from scrapy.utils.httpobj import urlparse_cached + + + class IPThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + host = urlparse_cached(request).hostname + address = await run_in_thread(socket.gethostbyname, host) + return add_scope(scopes, address) .. _throttling-settings: @@ -1024,6 +1072,8 @@ API .. autoclass:: scrapy.throttling.ThrottlingScopeManager +.. autoclass:: scrapy.pqueues.ThrottlingAwarePriorityQueue + .. autoclass:: scrapy.throttling.ThrottlingScopeConfig .. autoclass:: scrapy.throttling.BackoffConfig diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 9096cff5a..7b761d80b 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -31,6 +31,7 @@ from scrapy.exceptions import ( from scrapy.http import Request, Response from scrapy.utils.asyncio import ( AsyncioLoopingCall, + call_later, create_looping_call, is_asyncio_available, ) @@ -57,6 +58,7 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings, Settings from scrapy.signalmanager import SignalManager from scrapy.spiders import Spider + from scrapy.utils.asyncio import CallLaterResult logger = logging.getLogger(__name__) @@ -132,6 +134,14 @@ class ExecutionEngine: # Requests currently held by the throttling manager, waiting for their # scopes to allow them through to the downloader. self._throttling_waiting: set[Request] = set() + # Number of in-flight asynchronous enqueue operations (see + # ``_enqueue_request_async``), so the spider is not considered idle + # while a request is still on its way into the scheduler. + self._scheduling: int = 0 + # A coalesced wakeup timer, armed when a throttling-aware scheduler + # reports that all pending requests are time-blocked (see + # ``next_request_delay``). + self._throttling_wakeup: CallLaterResult | None = None self._delayed_requests_warn_threshold: int = self.settings.getint( "DELAYED_REQUESTS_WARN_THRESHOLD" ) @@ -269,6 +279,7 @@ class ExecutionEngine: def pause(self) -> None: self.paused = True + self._cancel_throttling_wakeup() def unpause(self) -> None: self.paused = False @@ -338,13 +349,43 @@ class ExecutionEngine: if self._slot is None or self._slot.closing is not None or self.paused: return + self._cancel_throttling_wakeup() + while not self.needs_backout(): if not self._start_scheduled_request(): break + self._maybe_arm_throttling_wakeup() + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() + def _cancel_throttling_wakeup(self) -> None: + if self._throttling_wakeup is not None: + self._throttling_wakeup.cancel() + self._throttling_wakeup = None + + def _maybe_arm_throttling_wakeup(self) -> None: + """Arm a single coalesced wakeup when the scheduler is throttling-aware + and reports that every pending request is time-blocked. + + Concurrency-blocked states do not need this: a freed slot already + re-runs the loop from :meth:`_download`'s ``finally``. Only time-based + gates (delay, backoff, quota windows) need a timer, since nothing else + would re-run the loop while they are closed. + """ + assert self._slot is not None + scheduler = self._slot.scheduler + delay_fn = getattr(scheduler, "next_request_delay", None) + if delay_fn is None or not scheduler.has_pending_requests(): + return + delay = delay_fn() + if delay is None: + return + self._throttling_wakeup = call_later( + max(0.0, delay), self._slot.nextcall.schedule + ) + def needs_backout(self) -> bool: """Returns ``True`` if no more requests can be sent at the moment, or ``False`` otherwise. @@ -382,10 +423,18 @@ class ExecutionEngine: if len(self._throttling_waiting) < self._delayed_requests_warn_threshold: return self._delayed_requests_warned = True + recommendation = "" + # A throttling-aware scheduler holds throttled requests in the + # scheduler instead of in _throttling_waiting, so it does not hit this + # path; recommend it only when it is not already in use. + scheduler = self._slot.scheduler if self._slot is not None else None + if scheduler is None or not hasattr(scheduler, "next_request_delay"): + recommendation = ( + " Consider switching to scrapy.core.scheduler.ThrottlingAwareScheduler." + ) logger.warning( - f"There are {len(self._throttling_waiting)} requests held back by throttling. This may " - "indicate a throttling configuration that is too aggressive or a " - "site that is rate-limiting heavily. See DELAYED_REQUESTS_WARN_THRESHOLD.", + f"There are {len(self._throttling_waiting)} requests held back by " + f"throttling. See DELAYED_REQUESTS_WARN_THRESHOLD.{recommendation}", extra={"spider": self.spider}, ) @@ -464,6 +513,8 @@ class ExecutionEngine: return False if self._throttling_waiting: # requests held by the throttling manager return False + if self._scheduling: # requests still on their way into the scheduler + return False if self._start is not None: # not all start requests are handled return False return not self._slot.scheduler.has_pending_requests() @@ -476,6 +527,7 @@ class ExecutionEngine: self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request) -> None: + assert self._slot is not None # typing request_scheduled_result = self.signals.send_catch_log( signals.request_scheduled, request=request, @@ -485,11 +537,35 @@ class ExecutionEngine: for _, result in request_scheduled_result: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return - if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr] + scheduler = self._slot.scheduler + if hasattr(scheduler, "enqueue_request_async"): + self._scheduling += 1 + _schedule_coro(self._enqueue_request_async(request)) + return + if not scheduler.enqueue_request(request): self.signals.send_catch_log( signals.request_dropped, request=request, spider=self.spider ) + async def _enqueue_request_async(self, request: Request) -> None: + assert self._slot is not None + try: + stored = await self._slot.scheduler.enqueue_request_async(request) # type: ignore[attr-defined] + if not stored: + self.signals.send_catch_log( + signals.request_dropped, request=request, spider=self.spider + ) + except Exception: + logger.error( + "Error while enqueuing request", + exc_info=True, + extra={"spider": self.spider}, + ) + finally: + self._scheduling -= 1 + if self._slot is not None: + self._slot.nextcall.schedule() + def download(self, request: Request) -> Deferred[Response]: """Return a Deferred which fires with a Response as result, only downloader middlewares are applied""" warnings.warn( @@ -675,6 +751,8 @@ class ExecutionEngine: "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) + self._cancel_throttling_wakeup() + try: await self._slot.close() except Exception: diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 7217da942..add1891db 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred # noqa: TC002 from scrapy.spiders import Spider # noqa: TC001 +from scrapy.throttling import iter_scopes from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -25,6 +26,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.pqueues import ScrapyPriorityQueue from scrapy.statscollectors import StatsCollector + from scrapy.throttling import ThrottlingManagerProtocol logger = logging.getLogger(__name__) @@ -496,3 +498,109 @@ class Scheduler(BaseScheduler): def _write_dqs_state(self, dqdir: str, state: Any) -> None: with Path(dqdir, "active.json").open("w", encoding="utf-8") as f: json.dump(state, f) + + +class ThrottlingAwareScheduler(Scheduler): + """A :class:`Scheduler` that only ever hands the engine requests whose + :ref:`throttling scopes ` allow them to be sent **right + now**. + + This avoids the head-of-line blocking that the default scheduler can suffer + when a crawl mixes heavily-throttled scopes with unthrottled ones: with the + default scheduler, requests taken from the scheduler in order wait at the + throttling gate + (:meth:`~scrapy.throttling.ThrottlingManagerProtocol.acquire`) while + holding a concurrency slot, so enough throttled requests waiting on a clock + can fill :setting:`CONCURRENT_REQUESTS` and starve unthrottled requests + that could be sent right away. Requests held back by this scheduler instead + occupy neither concurrency slots nor memory (beyond what is needed to track + each distinct scope set), so they cannot starve other scopes. + + When several requests could be sent at the same time, the one with the + highest request :attr:`~scrapy.Request.priority` is sent first; ties are + broken by preferring the least-busy scopes. + + It requires :setting:`SCHEDULER_PRIORITY_QUEUE` to be set to + :class:`~scrapy.pqueues.ThrottlingAwarePriorityQueue` (or a compatible + subclass), and the configured :setting:`SCHEDULER_DISK_QUEUE` / + :setting:`SCHEDULER_MEMORY_QUEUE` to support ``peek``. + """ + + def open(self, spider: Spider) -> Deferred[None] | None: + result = super().open(spider) + if not hasattr(self.mqs, "next_request_delay"): + raise ValueError( + f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be " + f"set to a throttling-aware priority queue such as " + f"scrapy.pqueues.ThrottlingAwarePriorityQueue, but the " + f"configured one ({type(self.mqs).__name__}) is not." + ) + assert self.crawler is not None + assert self.crawler.throttler is not None + self._throttler: ThrottlingManagerProtocol = self.crawler.throttler + return result + + def enqueue_request(self, request: Request) -> bool: + raise RuntimeError( + "ThrottlingAwareScheduler requires the asynchronous enqueue path; " + "enqueue_request_async() is used by the engine instead of " + "enqueue_request()." + ) + + async def enqueue_request_async(self, request: Request) -> bool: + """Resolve the :ref:`throttling scope set ` of + *request* and store it under the queue for that scope set. + + This is the asynchronous counterpart of + :meth:`Scheduler.enqueue_request`; scope resolution + (:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scopes`) is + asynchronous, which is why enqueuing has to be too. + """ + if not request.dont_filter and self.df.request_seen(request): + self.df.log(request, self.spider) + return False + scope_set = frozenset(iter_scopes(await self._throttler.get_scopes(request))) + assert self.stats is not None + if self._dqpush_throttling(request, scope_set): + self.stats.inc_value("scheduler/enqueued/disk") + else: + self.mqs.push(request, scope_set) # type: ignore[call-arg] + self.stats.inc_value("scheduler/enqueued/memory") + self.stats.inc_value("scheduler/enqueued") + return True + + def _dqpush_throttling(self, request: Request, scope_set: frozenset[str]) -> bool: + if self.dqs is None: + return False + try: + self.dqs.push(request, scope_set) # type: ignore[call-arg] + except ValueError as e: # non serializable request + if self.logunser: + logger.warning( + "Unable to serialize request: %(request)s - reason:" + " %(reason)s - no more unserializable requests will be" + " logged (stats being collected)", + {"request": request, "reason": e}, + exc_info=True, + extra={"spider": self.spider}, + ) + self.logunser = False + assert self.stats is not None + self.stats.inc_value("scheduler/unserializable") + return False + return True + + def next_request_delay(self) -> float | None: + """Return the minimum number of seconds until some pending request + becomes sendable because a time-based throttling gate opens, or + ``None`` if no pending request is time-blocked. + + The engine uses this to arm a single wakeup timer when + :meth:`next_request` returns ``None`` while requests are still + pending.""" + delays = [ + delay + for pq in (self.mqs, self.dqs) + if pq is not None and (delay := pq.next_request_delay()) is not None # type: ignore[attr-defined] + ] + return min(delays) if delays else None diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 0ad0b5d78..60675fe7f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import json import logging from typing import TYPE_CHECKING, Protocol, cast @@ -15,6 +16,7 @@ if TYPE_CHECKING: from scrapy import Request from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler + from scrapy.throttling import ScopeID, ThrottlingManagerProtocol logger = logging.getLogger(__name__) @@ -437,3 +439,179 @@ class DownloaderAwarePriorityQueue: def __contains__(self, slot: str) -> bool: return slot in self.pqueues + + +def _scope_set_key(scope_set: frozenset[ScopeID]) -> str: + """Return a reversible, JSON-safe string key for *scope_set*. + + Used both as the in-memory dict key encoding for the on-disk state and to + derive the per-scope-set subdirectory name. The encoding is + order-independent (the scope ids are sorted).""" + return json.dumps(sorted(scope_set)) + + +def _scope_set_from_key(key: str) -> frozenset[ScopeID]: + return frozenset(json.loads(key)) + + +class ThrottlingAwarePriorityQueue: + """Priority queue that partitions requests by their full :ref:`throttling + scope set ` and only ever pops a request that can be + sent right now. + + The downstream queue class must support ``peek``. + + Disk persistence + ================ + + .. warning:: The files that this class generates on disk are an + implementation detail, and may change without a warning in a future + version of Scrapy. Do not rely on the following information for + anything other than debugging purposes. + + When instantiated with a non-empty *key* argument, *key* is used as a + persistence directory, and inside it this class creates a subdirectory per + scope set, named from a path-safe, order-independent encoding of its scope + ids. + """ + + @classmethod + def from_crawler( + cls, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, + ) -> Self: + return cls( + crawler, + downstream_queue_cls, + key, + startprios, + start_queue_cls=start_queue_cls, + ) + + def __init__( + self, + crawler: Crawler, + downstream_queue_cls: type[QueueProtocol], + key: str, + slot_startprios: dict[str, Iterable[int]] | None = None, + *, + start_queue_cls: type[QueueProtocol] | None = None, + ): + if slot_startprios and not isinstance(slot_startprios, dict): + raise ValueError( + "ThrottlingAwarePriorityQueue accepts ``slot_startprios`` as a " + f"dict; {slot_startprios.__class__!r} instance is passed. Most " + "likely, it means the state is created by an incompatible " + "priority queue. Only a crawl started with the same priority " + "queue class can be resumed." + ) + + assert crawler.throttler is not None + self._throttler: ThrottlingManagerProtocol = crawler.throttler + self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls + self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls + self.key: str = key + self.crawler: Crawler = crawler + + # scope set -> priority queue + self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {} + if slot_startprios: + for set_key, startprios in slot_startprios.items(): + scope_set = _scope_set_from_key(set_key) + self.pqueues[scope_set] = self.pqfactory(scope_set, startprios) + + def pqfactory( + self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = () + ) -> ScrapyPriorityQueue: + return ScrapyPriorityQueue( + self.crawler, + self.downstream_queue_cls, + self.key + "/" + _path_safe(_scope_set_key(scope_set)), + startprios, + start_queue_cls=self._start_queue_cls, + ) + + def push(self, request: Request, scope_set: frozenset[ScopeID]) -> None: + if scope_set not in self.pqueues: + self.pqueues[scope_set] = self.pqfactory(scope_set) + self.pqueues[scope_set].push(request) + + def _select( + self, + ) -> tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None: + """Return the sendable ``(scope_set, queue)`` pair to pop from, or + ``None`` if no queue can be popped from right now. + + Among the sendable queues (those whose scope set can be sent right now), + the one whose head has the highest request priority is chosen; ties are + broken by ascending load (the maximum + :meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the + scopes of the queue), i.e. by preferring the least-busy scopes. + """ + best_sort_key: tuple[int, float] | None = None + best: tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None = None + for scope_set, queue in self.pqueues.items(): + head = queue.peek() + if head is None or not self._throttler.is_ready(head): + continue + load = max( + (self._throttler.scope_load(scope_id) for scope_id in scope_set), + default=0.0, + ) + sort_key = (queue.priority(head), load) + if best_sort_key is None or sort_key < best_sort_key: + best_sort_key = sort_key + best = (scope_set, queue) + return best + + def pop(self) -> Request | None: + selected = self._select() + if selected is None: + return None + scope_set, queue = selected + request = queue.pop() + if request is not None: + self._throttler.reserve(request) + if len(queue) == 0: + del self.pqueues[scope_set] + return request + + def peek(self) -> Request | None: + selected = self._select() + if selected is None: + return None + return selected[1].peek() + + def next_request_delay(self) -> float | None: + delay: float | None = None + for queue in self.pqueues.values(): + head = queue.peek() + if head is None: + continue + if self._throttler.is_ready(head): + return 0.0 + head_delay = self._throttler.time_until_ready(head) + if head_delay is None: + continue + if delay is None or head_delay < delay: + delay = head_delay + return delay + + def close(self) -> dict[str, list[int]]: + active = { + _scope_set_key(scope_set): queue.close() + for scope_set, queue in self.pqueues.items() + } + self.pqueues.clear() + return active + + def __len__(self) -> int: + return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + + def __contains__(self, scope_set: frozenset[ScopeID]) -> bool: + return scope_set in self.pqueues diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 9b70ec23f..de6589796 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -317,6 +317,48 @@ class ThrottlingManagerProtocol(Protocol): enforce a concurrency limit can let other requests through. """ + def is_ready(self, request: Request) -> bool: + """Return whether every scope of *request* allows it to be sent right + now, i.e. all time-based gates (delay, backoff, quota window) are open + *and* a concurrency slot is free in every scope. + + This is the synchronous, non-blocking counterpart of :meth:`acquire`, + used by a :ref:`throttling-aware scheduler ` + to decide whether a request can be dequeued now. It assumes the scopes + of *request* have already been resolved (e.g. by an earlier + :meth:`get_scopes` call at enqueue time). + """ + + def reserve(self, request: Request) -> None: + """Claim a send for *request*: record the send on every one of its + scopes and mark *request* as reserved, so that a later :meth:`acquire` + for it returns immediately without reserving again. + + A :ref:`throttling-aware scheduler ` calls + this when it decides to dequeue *request* (after :meth:`is_ready` + returned ``True``). The reservation is released by :meth:`release`. + """ + + def time_until_ready(self, request: Request) -> float | None: + """Return the number of seconds until every time-based gate of + *request* would be open, or ``None`` if no time-based gate is currently + blocking it (only a concurrency slot could be). + + Used by a :ref:`throttling-aware scheduler + ` to schedule a wakeup when all pending + requests are time-blocked. + """ + + def scope_load(self, scope_id: str) -> float: + """Return the current load of the scope identified by *scope_id*: its + active sends divided by its concurrency limit (or by the global + :setting:`CONCURRENT_REQUESTS` when the scope has no explicit limit). + + Used by a :ref:`throttling-aware scheduler + ` to balance dequeuing across scopes, + preferring the least-loaded ones. + """ + async def process_response(self, response: Response) -> None: """Update the throttling state based on *response*.""" @@ -352,10 +394,14 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: async def get_scopes(self, request): return urlparse_cached(request).netloc """ - cache: WeakKeyDictionary[Request, RequestScopes] = WeakKeyDictionary() @wraps(f) async def wrapper(self: Any, request: Request) -> RequestScopes: + cache: WeakKeyDictionary[Request, RequestScopes] | None = self.__dict__.get( + "_scope_cache" + ) + if cache is None: + cache = self.__dict__["_scope_cache"] = WeakKeyDictionary() if request in cache: return cache[request] scopes = await f(self, request) @@ -405,6 +451,7 @@ class ThrottlingManager: "THROTTLING_SCOPES" ) self._scope_managers: dict[ScopeID, ThrottlingScopeManagerProtocol] = {} + self._global_concurrency: int = crawler.settings.getint("CONCURRENT_REQUESTS") self._last_eviction: float | None = None # Concurrency slots reserved by acquire(), to be released once the # request finishes downloading. @@ -414,11 +461,37 @@ class ThrottlingManager: @scope_cache async def get_scopes(self, request: Request) -> RequestScopes: + return self._resolve_scopes_sync(request) + + def _resolve_scopes_sync(self, request: Request) -> RequestScopes: + """Best-effort synchronous scope resolution. + + It mirrors :meth:`get_scopes`, and is used by the synchronous + readiness methods (:meth:`is_ready`, :meth:`reserve`, + :meth:`time_until_ready`) when the cached result of an earlier + :meth:`get_scopes` call is not available (e.g. for a request restored + from disk). Subclasses whose :meth:`get_scopes` cannot be resolved + synchronously should rely on the enqueue-time cache instead. + """ scopes = request.meta.get("throttling_scopes") if scopes is not None: return cast("RequestScopes", scopes) return urlparse_cached(request).netloc + def _cached_scope_values( + self, request: Request + ) -> list[tuple[ScopeID, float | None]]: + """Return the ``(scope_id, quota_amount)`` pairs of *request*, reading + the cache populated by :meth:`get_scopes` and falling back to + :meth:`_resolve_scopes_sync`.""" + cache: WeakKeyDictionary[Request, RequestScopes] | None = self.__dict__.get( + "_scope_cache" + ) + scopes = cache.get(request) if cache is not None else None + if scopes is None: + scopes = self._resolve_scopes_sync(request) + return list(iter_scope_values(scopes)) + async def get_initial_backoff(self) -> BackoffData: return None @@ -472,6 +545,10 @@ class ThrottlingManager: return manager async def acquire(self, request: Request) -> None: + # A throttling-aware scheduler reserves the request before handing it + # to the engine, so there is nothing left to wait for or record here. + if request in self._reserved: + return now = time.monotonic() self._maybe_evict(now) await self._apply_request_delay(request) @@ -518,6 +595,44 @@ class ThrottlingManager: for manager, _ in managers: manager.record_done() + # -- Synchronous readiness API (used by a throttling-aware scheduler) ------ + + def is_ready(self, request: Request) -> bool: + now = time.monotonic() + for scope_id, value in self._cached_scope_values(request): + manager = self._get_scope_manager(scope_id) + if manager.can_send(now=now, amount=value) > 0: + return False + if manager.concurrency_blocked(): + return False + return True + + def reserve(self, request: Request) -> None: + managers = [ + (self._get_scope_manager(scope_id), value) + for scope_id, value in self._cached_scope_values(request) + ] + for manager, value in managers: + manager.record_sent(amount=value) + self._reserved[request] = managers + + def time_until_ready(self, request: Request) -> float | None: + now = time.monotonic() + wait = 0.0 + for scope_id, value in self._cached_scope_values(request): + manager = self._get_scope_manager(scope_id) + wait = max(wait, manager.can_send(now=now, amount=value)) + return wait if wait > 0 else None + + def scope_load(self, scope_id: ScopeID) -> float: + manager = self._get_scope_manager(scope_id) + active: int = getattr(manager, "_active", 0) + concurrency: int | None = getattr(manager, "_concurrency", None) + limit = concurrency if concurrency is not None else self._global_concurrency + if not limit: + return 0.0 + return active / limit + async def _wait_for_slot(self, managers: list[Any]) -> None: """Block until any of *managers* frees a concurrency slot. @@ -749,13 +864,13 @@ class ThrottlingScopeManagerProtocol(Protocol): Return ``False`` when no concurrency limit is enforced. """ - def slot_event(self) -> Deferred: + def slot_event(self) -> Deferred[None]: """Return a :class:`~twisted.internet.defer.Deferred` that fires when a concurrency slot next becomes available (e.g. when :meth:`record_done` is called or the limit is raised via :meth:`set_concurrency`).""" - def discard_slot_event(self, event: Deferred) -> None: + def discard_slot_event(self, event: Deferred[None]) -> None: """Cancel a pending slot event returned by :meth:`slot_event`. Called by :class:`ThrottlingManager` when the wait ends without the diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 6c6a6584a..9843ca4d5 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -6,12 +6,18 @@ import queuelib from scrapy.core.downloader import Downloader from scrapy.http.request import Request -from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue +from scrapy.pqueues import ( + DownloaderAwarePriorityQueue, + ScrapyPriorityQueue, + ThrottlingAwarePriorityQueue, +) from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue +from scrapy.throttling import iter_scopes from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader +from tests.utils.decorators import coroutine_test class TestPriorityQueue: @@ -309,3 +315,107 @@ def test_pop_order(input_, output): actual_output_urls.append(request.url) assert actual_output_urls == expected_output_urls + + +class TestThrottlingAwarePriorityQueue: + def _queue(self, crawler, key=""): + return build_from_crawler( + ThrottlingAwarePriorityQueue, + crawler, + downstream_queue_cls=FifoMemoryQueue, + key=key, + ) + + async def _push(self, queue, crawler, request): + scope_set = frozenset(iter_scopes(await crawler.throttler.get_scopes(request))) + queue.push(request, scope_set) + + @coroutine_test + async def test_partitions_by_scope_set(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://a.com/1")) + await self._push(queue, crawler, Request("http://a.com/2")) + await self._push(queue, crawler, Request("http://b.com/1")) + # Two distinct scope sets -> two internal queues, three requests. + assert len(queue.pqueues) == 2 + assert len(queue) == 3 + + @coroutine_test + async def test_pop_skips_blocked_scope(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLING_SCOPES": {"slow.com": {"delay": 1000.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + }, + ) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://slow.com/1")) + await self._push(queue, crawler, Request("http://slow.com/2")) + await self._push(queue, crawler, Request("http://fast.com/1")) + # The first slow request is sendable (no delay accrued yet); after it is + # popped (and reserved), the second slow request is blocked, but the + # fast one is still served. + popped = [queue.pop(), queue.pop(), queue.pop()] + urls = [r.url if r else None for r in popped] + assert "http://fast.com/1" in urls + assert "http://slow.com/1" in urls + # The blocked second slow request stays in the queue. + assert None in urls + assert len(queue) == 1 + delay = queue.next_request_delay() + assert delay is not None + assert delay == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_least_loaded_first(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLING_SCOPES": { + "a.com": {"concurrency": 4}, + "b.com": {"concurrency": 4}, + } + }, + ) + queue = self._queue(crawler) + # Make a.com busier than b.com. + busy = Request("http://a.com/0") + await crawler.throttler.get_scopes(busy) + crawler.throttler.reserve(busy) + await self._push(queue, crawler, Request("http://a.com/1")) + await self._push(queue, crawler, Request("http://b.com/1")) + # Equal priority, so the lower-load scope (b.com) is served first. + assert queue.pop().url == "http://b.com/1" + + @coroutine_test + async def test_priority_beats_load(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLING_SCOPES": { + "a.com": {"concurrency": 4}, + "b.com": {"concurrency": 4}, + } + }, + ) + queue = self._queue(crawler) + # Make a.com busier than b.com. + busy = Request("http://a.com/0") + await crawler.throttler.get_scopes(busy) + crawler.throttler.reserve(busy) + # The a.com request has higher priority, and a.com still has room, so it + # is served first despite a.com being the busier scope. + await self._push(queue, crawler, Request("http://a.com/1", priority=10)) + await self._push(queue, crawler, Request("http://b.com/1")) + assert queue.pop().url == "http://a.com/1" + + @coroutine_test + async def test_empty_and_close(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + assert queue.pop() is None + assert queue.next_request_delay() is None + await self._push(queue, crawler, Request("http://a.com/1")) + assert queue.close() != {} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8637de41e..9919b05a0 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -10,7 +10,7 @@ from unittest.mock import Mock import pytest from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import BaseScheduler, Scheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler, ThrottlingAwareScheduler from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -406,3 +406,175 @@ class TestIncompatibility: ValueError, match="does not support CONCURRENT_REQUESTS_PER_IP" ): self._incompatible() + + +_THROTTLING_AWARE_PQ = "scrapy.pqueues.ThrottlingAwarePriorityQueue" + + +class TestThrottlingAwareScheduler: + def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler: + settings = { + "SCHEDULER_PRIORITY_QUEUE": _THROTTLING_AWARE_PQ, + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + **(settings_dict or {}), + } + return get_crawler(Spider, settings) + + def _scheduler(self, crawler: Crawler) -> ThrottlingAwareScheduler: + spider = Spider(name="spider") + crawler.spider = spider + scheduler = ThrottlingAwareScheduler.from_crawler(crawler) + scheduler.open(spider) + return scheduler + + @coroutine_test + async def test_enqueue_async_and_dequeue(self) -> None: + scheduler = self._scheduler(self._crawler()) + assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True + assert scheduler.has_pending_requests() + assert len(scheduler) == 1 + request = scheduler.next_request() + assert request is not None + assert request.url == "http://a.com/1" + assert scheduler.next_request() is None + assert not scheduler.has_pending_requests() + scheduler.close("finished") + + def test_sync_enqueue_raises(self) -> None: + scheduler = self._scheduler(self._crawler()) + with pytest.raises(RuntimeError, match="asynchronous enqueue path"): + scheduler.enqueue_request(Request("http://a.com/1")) + scheduler.close("finished") + + def test_requires_throttling_aware_priority_queue(self) -> None: + crawler = self._crawler( + {"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ScrapyPriorityQueue"} + ) + spider = Spider(name="spider") + crawler.spider = spider + scheduler = ThrottlingAwareScheduler.from_crawler(crawler) + with pytest.raises(ValueError, match="throttling-aware priority queue"): + scheduler.open(spider) + + @coroutine_test + async def test_delay_blocks_and_reports_delay(self) -> None: + crawler = self._crawler( + { + "THROTTLING_SCOPES": {"slow.com": {"delay": 1000.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + }, + ) + scheduler = self._scheduler(crawler) + await scheduler.enqueue_request_async(Request("http://slow.com/1")) + await scheduler.enqueue_request_async(Request("http://slow.com/2")) + # The first request is sendable; the second is blocked by the delay. + first = scheduler.next_request() + assert first is not None + assert scheduler.next_request() is None + assert scheduler.has_pending_requests() + assert scheduler.next_request_delay() == pytest.approx(1000.0, abs=1.0) + scheduler.close("finished") + + @coroutine_test + async def test_no_delay_when_only_concurrency_blocked(self) -> None: + crawler = self._crawler( + {"THROTTLING_SCOPES": {"slow.com": {"concurrency": 1}}}, + ) + scheduler = self._scheduler(crawler) + await scheduler.enqueue_request_async(Request("http://slow.com/1")) + await scheduler.enqueue_request_async(Request("http://slow.com/2")) + assert scheduler.next_request() is not None + assert scheduler.next_request() is None + # A purely concurrency-blocked state has no time-based wakeup. + assert scheduler.next_request_delay() is None + scheduler.close("finished") + + @coroutine_test + async def test_resume_from_disk(self, tmp_path: Path) -> None: + settings = {"JOBDIR": str(tmp_path)} + scheduler = self._scheduler(self._crawler(settings)) + await scheduler.enqueue_request_async(Request("http://a.com/1")) + await scheduler.enqueue_request_async(Request("http://b.com/1")) + scheduler.close("shutdown") + + scheduler2 = self._scheduler(self._crawler(settings)) + assert len(scheduler2) == 2 + urls = set() + while (request := scheduler2.next_request()) is not None: + urls.add(request.url) + assert urls == {"http://a.com/1", "http://b.com/1"} + scheduler2.close("finished") + + +class TestIntegrationWithThrottlingAwareScheduler: + @inline_callbacks_test + def test_integration(self): + crawler = get_crawler( + spidercls=StartUrlsSpider, + settings_dict={ + "SCHEDULER": "scrapy.core.scheduler.ThrottlingAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlingAwarePriorityQueue", + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + }, + ) + with MockServer() as mockserver: + url = mockserver.url("/status?n=200", is_secure=False) + start_urls = [url] * 6 + yield crawler.crawl(start_urls) + assert crawler.stats.get_value("downloader/response_count") == len( + start_urls + ) + + @inline_callbacks_test + def test_integration_follow_requests(self): + # Exercises the engine's asynchronous enqueue path for requests yielded + # from a callback (not just start requests), and the idle guard that + # keeps the spider open while an enqueue is in flight. + class FollowSpider(Spider): + name = "follow" + + def __init__(self, base_url, **kwargs): + self.base_url = base_url + super().__init__(**kwargs) + + async def start(self): + yield Request(self.base_url + "/status?n=200") + + def parse(self, response): + if b"follow" not in response.url.encode(): + yield Request(response.url + "&follow=1", callback=self.parse) + + with MockServer() as mockserver: + base_url = mockserver.url("", is_secure=False).rstrip("/") + crawler = get_crawler( + spidercls=FollowSpider, + settings_dict={ + "SCHEDULER": "scrapy.core.scheduler.ThrottlingAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlingAwarePriorityQueue", + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + }, + ) + yield crawler.crawl(base_url=base_url) + assert crawler.stats.get_value("downloader/response_count") == 2 + + @inline_callbacks_test + def test_integration_with_delay(self): + # A small per-scope delay forces the engine to arm the throttling wakeup + # timer between requests; the crawl must still complete. + crawler = get_crawler( + spidercls=StartUrlsSpider, + settings_dict={ + "SCHEDULER": "scrapy.core.scheduler.ThrottlingAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlingAwarePriorityQueue", + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "RANDOMIZE_DOWNLOAD_DELAY": False, + "THROTTLING_SCOPES": {"127.0.0.1": {"delay": 0.05}}, + }, + ) + with MockServer() as mockserver: + url = mockserver.url("/status?n=200", is_secure=False) + start_urls = [url] * 4 + yield crawler.crawl(start_urls) + assert crawler.stats.get_value("downloader/response_count") == len( + start_urls + ) diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 0e0269c95..bd71ac255 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -540,6 +540,91 @@ class TestThrottlingScopeManager: assert scope._concurrency == 1 +class TestThrottlingManagerReadiness: + """The synchronous readiness API used by a throttling-aware scheduler.""" + + @coroutine_test + async def test_is_ready_unconstrained_scope(self): + manager = _manager() + request = Request("http://example.com/a") + # A scope with no configured delay/concurrency/quota is always ready. + assert await manager.get_scopes(request) == "example.com" + assert manager.is_ready(request) is True + + @coroutine_test + async def test_is_ready_without_cached_scopes(self): + # is_ready falls back to synchronous resolution when get_scopes was not + # called first (e.g. for a request restored from disk). + manager = _manager() + request = Request("http://example.com/a") + assert manager.is_ready(request) is True + + @coroutine_test + async def test_reserve_blocks_delay_scope(self): + manager = _manager( + { + "THROTTLING_SCOPES": {"example.com": {"delay": 100.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + } + ) + first = Request("http://example.com/1") + second = Request("http://example.com/2") + await manager.get_scopes(first) + await manager.get_scopes(second) + assert manager.is_ready(first) is True + manager.reserve(first) + # The base delay now blocks any further request for the scope. + assert manager.is_ready(second) is False + assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0) + + @coroutine_test + async def test_reserve_blocks_on_concurrency(self): + manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) + first = Request("http://example.com/1") + second = Request("http://example.com/2") + await manager.get_scopes(first) + await manager.get_scopes(second) + manager.reserve(first) + assert manager.is_ready(second) is False + # Pure concurrency blocking is not time-gated. + assert manager.time_until_ready(second) is None + manager.release(first) + assert manager.is_ready(second) is True + + @coroutine_test + async def test_acquire_noop_when_reserved(self): + manager = _manager( + { + "THROTTLING_SCOPES": {"example.com": {"delay": 100.0}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + } + ) + request = Request("http://example.com/1") + await manager.get_scopes(request) + manager.reserve(request) + # A reserved request fast-paths through acquire() without re-recording + # the send or waiting for the delay. + await manager.acquire(request) + scope = manager._get_scope_manager("example.com") + assert scope._active == 1 # reserve recorded exactly one send + + @coroutine_test + async def test_scope_load(self): + manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 4}}}) + assert manager.scope_load("example.com") == 0.0 + request = Request("http://example.com/1") + await manager.get_scopes(request) + manager.reserve(request) + assert manager.scope_load("example.com") == pytest.approx(0.25) + + def test_scope_load_falls_back_to_global_concurrency(self): + manager = _manager({"CONCURRENT_REQUESTS": 8}) + # A scope with no explicit concurrency limit uses CONCURRENT_REQUESTS. + request = Request("http://example.com/1") + manager.reserve(request) + assert manager.scope_load("example.com") == pytest.approx(1 / 8) + + class TestThrottlingIntegration: @coroutine_test async def test_backoff_recorded_on_429(self, mockserver): From 6ce90dde3e5d6190b4b1faeece360e82c1bfd6c4 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 12:15:52 +0200 Subject: [PATCH 12/55] =?UTF-8?q?Minimum=20queuelib:=201.4.2=20=E2=86=92?= =?UTF-8?q?=201.6.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/throttling.rst | 4 ++++ pyproject.toml | 2 +- scrapy/core/scheduler.py | 29 +++++++++++++++++++++++++++++ tests/test_scheduler.py | 23 +++++++++++++++++++++++ tox.ini | 4 ++-- 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index cc8519990..332a874c5 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -331,6 +331,10 @@ apply different throttling based on request priority. Use the ``throttling_scopes`` request metadata to assign requests to custom throttling groups: +.. invisible-code-block: python + + from scrapy.http import Request + .. code-block:: python Request("https://api.example/", meta={"throttling_scopes": "api"}) diff --git a/pyproject.toml b/pyproject.toml index 6b0c561f8..e3d1b9989 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ "parsel>=1.5.0", "protego>=0.1.15", "pyOpenSSL>=22.0.0", - "queuelib>=1.4.2", + "queuelib>=1.6.1", "service_identity>=23.1.0", "tldextract", "w3lib>=1.17.0", diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index add1891db..76eb3b5fe 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -500,6 +500,22 @@ class Scheduler(BaseScheduler): json.dump(state, f) +def _queue_supports_peek(queue_cls: type) -> bool: + """Return whether *queue_cls* (a Scrapy queue class) is backed by an + underlying queue that really implements ``peek``. + + Scrapy's queue wrappers in :mod:`scrapy.squeues` always define a ``peek`` + method, but it merely delegates to the underlying queue class (e.g. a + queuelib one, which only gained ``peek`` in queuelib 1.6.1) and raises + :exc:`NotImplementedError` if that one lacks it. This checks the real + implementation by ignoring the delegating wrappers. + """ + return any( + "peek" in base.__dict__ and base.__module__ != "scrapy.squeues" + for base in queue_cls.__mro__ + ) + + class ThrottlingAwareScheduler(Scheduler): """A :class:`Scheduler` that only ever hands the engine requests whose :ref:`throttling scopes ` allow them to be sent **right @@ -535,6 +551,19 @@ class ThrottlingAwareScheduler(Scheduler): f"scrapy.pqueues.ThrottlingAwarePriorityQueue, but the " f"configured one ({type(self.mqs).__name__}) is not." ) + for setting, pq in ( + ("SCHEDULER_MEMORY_QUEUE", self.mqs), + ("SCHEDULER_DISK_QUEUE", self.dqs), + ): + if pq is None: + continue + queue_cls = getattr(pq, "downstream_queue_cls", None) + if queue_cls is not None and not _queue_supports_peek(queue_cls): + raise ValueError( + f"{type(self).__name__} requires {setting} to be set to a " + f"queue class that supports peek(), but the configured one " + f"({queue_cls.__name__}) does not." + ) assert self.crawler is not None assert self.crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = self.crawler.throttler diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9919b05a0..94ef5b699 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -26,6 +26,9 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path + # typing.Self requires Python 3.11 + from typing_extensions import Self + class MemoryScheduler(BaseScheduler): paused = False @@ -411,6 +414,16 @@ class TestIncompatibility: _THROTTLING_AWARE_PQ = "scrapy.pqueues.ThrottlingAwarePriorityQueue" +class _NoPeekMemoryQueue: + """A memory queue class that does not implement ``peek``, used to check + that ThrottlingAwareScheduler rejects queues lacking peek support (e.g. when + queuelib is older than 1.6.1).""" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: + return cls() + + class TestThrottlingAwareScheduler: def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler: settings = { @@ -456,6 +469,16 @@ class TestThrottlingAwareScheduler: with pytest.raises(ValueError, match="throttling-aware priority queue"): scheduler.open(spider) + def test_requires_peek_supporting_queue(self) -> None: + crawler = self._crawler( + {"SCHEDULER_MEMORY_QUEUE": "tests.test_scheduler._NoPeekMemoryQueue"} + ) + spider = Spider(name="spider") + crawler.spider = spider + scheduler = ThrottlingAwareScheduler.from_crawler(crawler) + with pytest.raises(ValueError, match="supports peek"): + scheduler.open(spider) + @coroutine_test async def test_delay_blocks_and_reports_delay(self) -> None: crawler = self._crawler( diff --git a/tox.ini b/tox.ini index 2fa7b455d..770067b8f 100644 --- a/tox.ini +++ b/tox.ini @@ -143,7 +143,7 @@ deps = lxml==4.6.4 parsel==1.5.0 pyOpenSSL==22.0.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 w3lib==1.17.0 zope.interface==5.1.0 @@ -259,7 +259,7 @@ deps = lxml==5.3.2 parsel==1.5.0 pyOpenSSL==24.3.0 - queuelib==1.4.2 + queuelib==1.6.1 service_identity==23.1.0 # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses # an inline flag placement that Python 3.11 treats as an error: global From c889c1057d313adf3108403a86c590d8a5ceeee6 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 13:09:17 +0200 Subject: [PATCH 13/55] Improve test coverage --- tests/test_pqueues.py | 50 +++++ tests/test_robotstxt_interface.py | 13 ++ tests/test_scheduler.py | 45 +++++ tests/test_throttling.py | 300 +++++++++++++++++++++++++++++- tests/test_utils_asyncio.py | 24 +++ 5 files changed, 431 insertions(+), 1 deletion(-) diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 9843ca4d5..27c49ae41 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -419,3 +419,53 @@ class TestThrottlingAwarePriorityQueue: assert queue.next_request_delay() is None await self._push(queue, crawler, Request("http://a.com/1")) assert queue.close() != {} + + @coroutine_test + async def test_peek(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + assert queue.peek() is None + await self._push(queue, crawler, Request("http://a.com/1")) + head = queue.peek() + assert head is not None + assert head.url == "http://a.com/1" + # peek() does not consume the request. + assert len(queue) == 1 + + @coroutine_test + async def test_next_request_delay_zero_when_ready(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://a.com/1")) + # A sendable head means no wait is needed. + assert queue.next_request_delay() == 0.0 + + @coroutine_test + async def test_next_request_delay_ignores_empty_queues(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + # An empty (but still registered) internal queue is skipped. + queue.pqueues[frozenset({"a.com"})] = queue.pqfactory(frozenset({"a.com"})) + assert queue.next_request_delay() is None + + @coroutine_test + async def test_contains(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + scope_set = frozenset( + iter_scopes(await crawler.throttler.get_scopes(Request("http://a.com/1"))) + ) + assert scope_set not in queue + await self._push(queue, crawler, Request("http://a.com/1")) + assert scope_set in queue + + def test_non_dict_slot_startprios(self): + crawler = get_crawler(Spider) + with pytest.raises(ValueError, match="slot_startprios"): + build_from_crawler( + ThrottlingAwarePriorityQueue, + crawler, + downstream_queue_cls=FifoMemoryQueue, + key="", + startprios=[1, 2, 3], + ) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 5249736f2..1062c2d5c 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -88,6 +88,16 @@ class BaseRobotParserTest: assert rp.allowed("https://site.local/index.html", "*") assert rp.allowed("https://site.local/disallowed", "*") + def test_crawl_delay(self): + robotstxt_body = b"User-agent: *\nCrawl-delay: 10\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") == 10.0 + + def test_crawl_delay_unset(self): + robotstxt_body = b"User-agent: *\nDisallow: /private\n" + rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body) + assert rp.crawl_delay("*") is None + def test_unicode_url_and_useragent(self): robotstxt_robotstxt_body = """ User-Agent: * @@ -171,6 +181,9 @@ class TestRerpRobotParser(BaseRobotParserTest): def test_length_based_precedence(self): pytest.skip("Rerp does not support length based directives precedence.") + def test_crawl_delay(self): + pytest.skip("Rerp does not expose Crawl-delay directives.") + class TestProtegoRobotParser(BaseRobotParserTest): def setup_method(self): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 94ef5b699..03faa3325 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import warnings from abc import ABC, abstractmethod from collections import deque @@ -512,6 +513,50 @@ class TestThrottlingAwareScheduler: assert scheduler.next_request_delay() is None scheduler.close("finished") + @coroutine_test + async def test_enqueue_async_filters_duplicates(self) -> None: + crawler = self._crawler( + {"DUPEFILTER_CLASS": "scrapy.dupefilters.RFPDupeFilter"} + ) + scheduler = self._scheduler(crawler) + crawler.spider.crawler = crawler # the dupefilter logs via spider.crawler + assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True + # The same request is filtered out the second time around. + assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is False + assert len(scheduler) == 1 + scheduler.close("finished") + + @coroutine_test + async def test_enqueue_async_unserializable_falls_back_to_memory( + self, tmp_path: Path, caplog + ) -> None: + crawler = self._crawler({"JOBDIR": str(tmp_path), "SCHEDULER_DEBUG": True}) + scheduler = self._scheduler(crawler) + # A lambda callback cannot be serialized to disk, so the request falls + # back to the in-memory queue and the failure is logged once. + request = Request("http://a.com/1", callback=lambda response: None) + with caplog.at_level(logging.WARNING, logger="scrapy.core.scheduler"): + assert await scheduler.enqueue_request_async(request) is True + assert "Unable to serialize request" in caplog.text + assert crawler.stats is not None + assert crawler.stats.get_value("scheduler/unserializable") == 1 + assert crawler.stats.get_value("scheduler/enqueued/memory") == 1 + scheduler.close("finished") + + @coroutine_test + async def test_enqueue_async_unserializable_without_debug( + self, tmp_path: Path + ) -> None: + # Same fallback as above, but with SCHEDULER_DEBUG off the failure is + # tracked in stats without logging a warning. + crawler = self._crawler({"JOBDIR": str(tmp_path)}) + scheduler = self._scheduler(crawler) + request = Request("http://a.com/1", callback=lambda response: None) + assert await scheduler.enqueue_request_async(request) is True + assert crawler.stats is not None + assert crawler.stats.get_value("scheduler/unserializable") == 1 + scheduler.close("finished") + @coroutine_test async def test_resume_from_disk(self, tmp_path: Path) -> None: settings = {"JOBDIR": str(tmp_path)} diff --git a/tests/test_throttling.py b/tests/test_throttling.py index bd71ac255..e9fa15c63 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -7,7 +7,18 @@ import pytest from scrapy import signals from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response -from scrapy.throttling import ThrottlingManager, ThrottlingScopeManager +from scrapy.throttling import ( + ThrottlingManager, + ThrottlingScopeManager, + _add_bare_scope, + _parse_ratelimit_reset, + _parse_retry_after, + _to_scope_dict, + add_scope, + iter_scope_values, + iter_scopes, + update_scope_backoff, +) from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -625,6 +636,279 @@ class TestThrottlingManagerReadiness: assert manager.scope_load("example.com") == pytest.approx(1 / 8) +class TestParseRateHeaders: + @pytest.mark.parametrize( + ("value", "expected"), + [ + (b"\xff\xfe", None), # undecodable bytes + ("garbage-not-a-date", None), # neither a number nor a valid date + # A naive HTTP date (no timezone) in the past is treated as UTC and + # yields no positive delay. + ("Wed, 21 Oct 2015 07:28:00", None), + ], + ids=["undecodable", "garbage", "naive-past-date"], + ) + def test_parse_retry_after(self, value, expected): + assert _parse_retry_after(_response(headers={"Retry-After": value})) == expected + + @pytest.mark.parametrize( + "value", + [b"\xff\xfe", "not-a-number"], + ids=["undecodable", "non-numeric"], + ) + def test_parse_ratelimit_reset_invalid(self, value): + assert ( + _parse_ratelimit_reset(_response(headers={"RateLimit-Reset": value})) + is None + ) + + +class TestScopeHelpers: + def test_iter_scopes(self): + assert list(iter_scopes(None)) == [] + assert list(iter_scopes("a")) == ["a"] + assert list(iter_scopes({"a": 1.0, "b": 2.0})) == ["a", "b"] + assert list(iter_scopes(["a", "b"])) == ["a", "b"] + + def test_iter_scope_values(self): + assert list(iter_scope_values(None)) == [] + assert list(iter_scope_values("a")) == [("a", None)] + assert list(iter_scope_values({"a": 1.0})) == [("a", 1.0)] + assert list(iter_scope_values(["a", "b"])) == [("a", None), ("b", None)] + + def test_to_scope_dict(self): + assert _to_scope_dict({"a": 1}, list) == {"a": 1} + assert _to_scope_dict(None, list) == {} + assert _to_scope_dict("a", lambda: None) == {"a": None} + assert _to_scope_dict(["a", "b"], dict) == {"a": {}, "b": {}} + with pytest.raises(TypeError): + _to_scope_dict(123, dict) + + def test_add_bare_scope(self): + assert _add_bare_scope(None, "a", None) == "a" + assert _add_bare_scope("a", "a", None) == "a" + assert _add_bare_scope("a", "b", None) == {"a", "b"} + assert _add_bare_scope({"a": 1}, "b", {}) == {"a": 1, "b": {}} + assert _add_bare_scope({"a": 1}, "a", {}) == {"a": 1} + assert _add_bare_scope(["a"], "b", None) == {"a", "b"} + assert _add_bare_scope(["a"], "a", None) == ["a"] + with pytest.raises(TypeError): + _add_bare_scope(123, "a", None) + + def test_add_scope_without_value(self): + assert add_scope(None, "a") == "a" + assert add_scope("a", "b") == {"a", "b"} + + def test_add_scope_with_value(self): + assert add_scope(None, "a", 2.0) == {"a": 2.0} + assert add_scope({"a": None}, "b", 3.0) == {"a": None, "b": 3.0} + + def test_add_scope_with_value_rejects_non_dict_entry(self): + with pytest.raises(TypeError): + add_scope({"a": 1.0}, "a", 2.0) + + def test_update_scope_backoff_without_params(self): + assert update_scope_backoff(None, "a") == "a" + assert update_scope_backoff("a", "b") == {"a", "b"} + + def test_update_scope_backoff_with_params(self): + assert update_scope_backoff(None, "a", delay=5.0, consumed=2.0) == { + "a": {"delay": 5.0, "consumed": 2.0} + } + assert update_scope_backoff({"a": {}}, "a", delay=1.0) == {"a": {"delay": 1.0}} + assert update_scope_backoff(None, "a", consumed=2.0) == {"a": {"consumed": 2.0}} + + def test_update_scope_backoff_rejects_non_dict_entry(self): + with pytest.raises(TypeError): + update_scope_backoff({"a": 1.0}, "a", delay=1.0) + + +class TestThrottlingManagerEdges: + @coroutine_test + async def test_acquire_without_scopes(self): + manager = _manager() + request = Request("http://example.com/a", meta={"throttling_scopes": []}) + # No scopes resolve, so acquire() returns without reserving anything. + await manager.acquire(request) + assert request not in manager._reserved + + def test_scope_load_without_concurrency_limit(self): + manager = _manager({"CONCURRENT_REQUESTS": 0}) + # CONCURRENT_REQUESTS is 0, so the load denominator is 0 and the load is + # reported as 0 instead of raising. + assert manager.scope_load("example.com") == 0.0 + + @coroutine_test + async def test_acquire_logs_and_waits_for_delay(self): + manager = _manager( + { + "THROTTLING_SCOPES": {"example.com": {"delay": 0.02}}, + "RANDOMIZE_DOWNLOAD_DELAY": False, + "THROTTLING_DEBUG": True, + } + ) + r1 = Request("http://example.com/1") + r2 = Request("http://example.com/2") + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r1))) + # r2 must wait out the per-scope delay accrued by r1 before it proceeds. + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r2))) + assert r2 in manager._reserved + + @coroutine_test + async def test_acquire_logs_while_waiting_for_slot(self): + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + + manager = _manager( + { + "THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}, + "THROTTLING_DEBUG": True, + } + ) + r1 = Request("http://example.com/1") + r2 = Request("http://example.com/2") + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r1))) + call_later(0, manager.release, r1) + # r2 is concurrency-blocked and waits, with debug logging, for the slot. + await maybe_deferred_to_future(deferred_from_coro(manager.acquire(r2))) + assert r2 in manager._reserved + + @coroutine_test + async def test_apply_request_delay(self): + manager = _manager({"THROTTLING_DEBUG": True}) + request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) + await manager._apply_request_delay(request) + assert request.meta["_throttling_delayed"] is True + # A second call is a no-op (the request was already delayed). + await manager._apply_request_delay(request) + + @coroutine_test + async def test_process_exception_applies_backoff(self): + manager = _manager() + request = Request("http://example.com/a") + await manager.process_exception(request, DownloadTimeoutError()) + assert manager._get_scope_manager("example.com")._backoff_level == 1 + + def test_apply_backoff_debug_logging(self, caplog): + manager = _manager({"THROTTLING_DEBUG": True}) + with caplog.at_level(logging.DEBUG, logger="scrapy.throttling"): + manager._apply_backoff("example.com") + assert "Backoff for scope" in caplog.text + assert manager._get_scope_manager("example.com")._backoff_level == 1 + + def test_on_robots_parsed_disabled(self): + manager = _manager({"THROTTLING_ROBOTSTXT_OBEY": False}) + # With obeying disabled, the handler returns without touching any scope. + manager._on_robots_parsed(_FakeRobotParser(5.0), Request("http://example.com")) + assert not manager._scope_managers + + def test_apply_robots_crawl_delay_warns_on_delay_conflict(self, caplog): + manager = _manager( + { + "ROBOTSTXT_OBEY": True, + "THROTTLING_SCOPES": {"example.com": {"delay": 0.5}}, + } + ) + with caplog.at_level(logging.WARNING, logger="scrapy.throttling"): + manager.apply_robots_crawl_delay("example.com", 3.0) + assert "Crawl-delay" in caplog.text + + def test_apply_robots_crawl_delay_debug_logging(self, caplog): + manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLING_DEBUG": True}) + with caplog.at_level(logging.DEBUG, logger="scrapy.throttling"): + manager.apply_robots_crawl_delay("example.com", 3.0) + assert "Crawl-delay" in caplog.text + + def test_maybe_evict_disabled(self): + manager = _manager({"THROTTLING_SCOPE_MAX_IDLE": 0}) + scope = manager._get_scope_manager("example.com") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # Eviction disabled (max idle <= 0): the scope is kept regardless. + manager._maybe_evict(now=10_000.0) + assert "example.com" in manager._scope_managers + + +class TestThrottlingScopeManagerEdges: + def test_jitter_as_range(self): + scope = _scope_manager( + config={"id": "x", "backoff": {"jitter": [0.0, 0.0]}, "min_delay": 1.0} + ) + # A [low, high] jitter range of [0, 0] leaves the value unchanged. + assert scope._apply_jitter(4.0) == pytest.approx(4.0) + + def test_effective_delay_randomized(self): + scope = _scope_manager( + {"RANDOMIZE_DOWNLOAD_DELAY": True}, {"id": "x", "delay": 2.0} + ) + scope.record_sent(now=0.0) + # A randomized base delay lands within [0.5, 1.5] * delay. + assert 1.0 <= scope.can_send(now=0.0) <= 3.0 + + def test_rampup_target_as_range(self): + scope = _scope_manager( + config={"id": "x", "rampup": {"backoff_target": [1, 3]}}, + ) + assert scope._rampup_target == (1.0, 3.0) + + def test_record_done_without_active(self): + scope = _scope_manager(config={"id": "x"}) + # Calling record_done() with nothing in flight is a harmless no-op. + scope.record_done(now=0.0) + assert scope._active == 0 + + def test_reconcile_quota_no_change(self): + scope = _scope_manager(config={"id": "x", "quota": 10.0}) + scope.record_sent(now=0.0, amount=4.0) + # Neither consumed nor remaining given: the estimate is left untouched. + scope.reconcile_quota(now=0.0) + assert scope._consumed == pytest.approx(4.0) + + def test_set_base_delay_during_backoff(self): + scope = _scope_manager( + {"BACKOFF_MIN_DELAY": 1.0, "BACKOFF_JITTER": 0}, {"id": "x"} + ) + scope.record_backoff(now=0.0) + backoff_delay = scope._delay + # Raising the base delay mid-backoff updates the base but not the current + # (higher) backoff delay. + scope.set_base_delay(0.5) + assert scope._base_delay == 0.5 + assert scope._delay == backoff_delay + + def test_rampup_step_held_during_backoff(self): + scope = _scope_manager(config={"id": "x", "delay": 4.0, "rampup": True}) + scope._backoff_level = 1 + before = scope._delay + scope._rampup_step() + # A rampup probe is skipped while a backoff is in progress. + assert scope._delay == before + + def test_record_sent_clears_expired_backoff(self): + scope = _scope_manager(config={"id": "x"}) + scope.record_backoff(delay=5.0, now=0.0) + assert scope._in_backoff_until == pytest.approx(5.0) + scope.record_sent(now=10.0) + # The hard backoff window has passed, so it is cleared. + assert scope._in_backoff_until is None + + def test_reconcile_quota_without_quota(self): + scope = _scope_manager(config={"id": "x"}) + # No quota configured, so reconciliation is a no-op. + scope.reconcile_quota(consumed=5.0, now=0.0) + assert scope._consumed == 0.0 + + def test_is_idle_with_active_requests(self): + scope = _scope_manager(config={"id": "x"}) + scope.record_sent(now=0.0) + # An in-flight request keeps the scope from being evicted. + assert scope.is_idle(now=10_000.0, max_idle=1.0) is False + + def test_is_idle_when_never_used(self): + scope = _scope_manager(config={"id": "x"}) + # A scope that was never used (no last_seen) is idle. + assert scope.is_idle(now=0.0, max_idle=1.0) is True + + class TestThrottlingIntegration: @coroutine_test async def test_backoff_recorded_on_429(self, mockserver): @@ -638,6 +922,20 @@ class TestThrottlingIntegration: assert managers, "no throttling scope was created" assert any(manager._backoff_level >= 1 for manager in managers.values()) + @coroutine_test + async def test_backoff_recorded_on_download_error(self, mockserver): + crawler = get_crawler(SimpleSpider, {"RETRY_ENABLED": False}) + # A dropped connection raises a DownloadFailedError, which the engine + # routes through the throttling manager before re-raising. + await crawler.crawl_async( + mockserver.url("/drop?abort=1"), mockserver=mockserver + ) + throttler = crawler.throttler + assert throttler is not None + managers = throttler._scope_managers + assert managers, "no throttling scope was created" + assert any(manager._backoff_level >= 1 for manager in managers.values()) + @coroutine_test async def test_no_backoff_on_200(self, mockserver): crawler = get_crawler(SimpleSpider) diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 5532b4a31..76f7f95c7 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -13,6 +13,8 @@ from scrapy.utils.asyncio import ( AsyncioLoopingCall, _parallel_asyncio, is_asyncio_available, + sleep, + wait_for_first, ) from tests.utils.decorators import coroutine_test @@ -149,3 +151,25 @@ class TestAsyncioLoopingCall: with pytest.raises(TypeError): looping_call.start(0.1) assert not looping_call.running + + +class TestSleep: + @coroutine_test + async def test_sleep(self): + # A zero-second sleep completes without error under any reactor mode. + await sleep(0) + + +class TestWaitForFirst: + @coroutine_test + async def test_empty(self): + # No deferreds -> two empty sets, returned immediately. + assert await wait_for_first([]) == (set(), set()) + + @coroutine_test + async def test_returns_done_deferred(self): + fired: Deferred[None] = Deferred() + fired.callback(None) + done, pending = await wait_for_first([fired], timeout=1) + assert done == {fired} + assert pending == set() From 8caa1ac4db8dcc3669a88a811cc02346525becab Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 13:27:14 +0200 Subject: [PATCH 14/55] Fix typing issues --- tests/test_scheduler.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 03faa3325..ad91efcc7 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -30,6 +30,8 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.request import CallbackT + class MemoryScheduler(BaseScheduler): paused = False @@ -519,6 +521,7 @@ class TestThrottlingAwareScheduler: {"DUPEFILTER_CLASS": "scrapy.dupefilters.RFPDupeFilter"} ) scheduler = self._scheduler(crawler) + assert crawler.spider is not None crawler.spider.crawler = crawler # the dupefilter logs via spider.crawler assert await scheduler.enqueue_request_async(Request("http://a.com/1")) is True # The same request is filtered out the second time around. @@ -534,7 +537,9 @@ class TestThrottlingAwareScheduler: scheduler = self._scheduler(crawler) # A lambda callback cannot be serialized to disk, so the request falls # back to the in-memory queue and the failure is logged once. - request = Request("http://a.com/1", callback=lambda response: None) + request = Request( + "http://a.com/1", callback=cast("CallbackT", lambda response: None) + ) with caplog.at_level(logging.WARNING, logger="scrapy.core.scheduler"): assert await scheduler.enqueue_request_async(request) is True assert "Unable to serialize request" in caplog.text @@ -551,7 +556,9 @@ class TestThrottlingAwareScheduler: # tracked in stats without logging a warning. crawler = self._crawler({"JOBDIR": str(tmp_path)}) scheduler = self._scheduler(crawler) - request = Request("http://a.com/1", callback=lambda response: None) + request = Request( + "http://a.com/1", callback=cast("CallbackT", lambda response: None) + ) assert await scheduler.enqueue_request_async(request) is True assert crawler.stats is not None assert crawler.stats.get_value("scheduler/unserializable") == 1 From 4a7a2d5b5ff04fbbadad220200e54727b69f3e10 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 14:39:13 +0200 Subject: [PATCH 15/55] Improve test coverage --- scrapy/throttling.py | 4 +- tests/test_core_downloader.py | 27 +++++- tests/test_engine.py | 155 ++++++++++++++++++++++++++++++++++ tests/test_pqueues.py | 37 ++++++++ tests/test_throttling.py | 46 ++++++++++ tests/test_utils_asyncio.py | 17 ++++ 6 files changed, 284 insertions(+), 2 deletions(-) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index de6589796..b46e192fa 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1054,7 +1054,9 @@ class ThrottlingScopeManager: self._rampup_min_delay, self._delay * self._rampup_delay_factor ) self._base_delay = min(self._base_delay, self._delay) - elif self._concurrency is not None: + else: + # Rampup is only enabled with a concurrency limit set. + assert self._concurrency is not None self._concurrency += 1 def _maybe_reset_quota(self, now: float) -> None: diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index fdd5edc27..cb14b588d 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,7 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy.core.downloader import Downloader, Slot, tls +from scrapy.core.downloader import Downloader, Slot, _get_concurrency_delay, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, _ScrapyClientContextFactory, @@ -45,6 +45,31 @@ class TestSlot: assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)" +class TestGetConcurrencyDelay: + def test_default(self): + crawler = get_crawler() + concurrency, delay = _get_concurrency_delay( + 8, DefaultSpider(), crawler.settings + ) + assert (concurrency, delay) == (8, 0.0) + + def test_spider_download_delay(self): + crawler = get_crawler() + spider = DefaultSpider() + spider.download_delay = 2.5 + _concurrency, delay = _get_concurrency_delay(8, spider, crawler.settings) + assert delay == 2.5 + + def test_download_delay_per_slot(self): + crawler = get_crawler(settings_dict={"DOWNLOAD_DELAY_PER_SLOT": 3.0}) + # DOWNLOAD_DELAY_PER_SLOT takes precedence over both DOWNLOAD_DELAY and + # the spider's download_delay attribute. + spider = DefaultSpider() + spider.download_delay = 2.5 + _concurrency, delay = _get_concurrency_delay(8, spider, crawler.settings) + assert delay == 3.0 + + @pytest.mark.requires_reactor # this test is related to the Twisted HTTP code class TestContextFactoryBase: @async_yield_fixture diff --git a/tests/test_engine.py b/tests/test_engine.py index e51eb4664..148a7f4e7 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -595,6 +595,21 @@ class TestEngineDownloadAsync: engine._slot.add_request.assert_called_once_with(request) engine._slot.remove_request.assert_called_once_with(request) + @coroutine_test + async def test_download_async_fetch_needs_spider(self, engine): + """A downloader whose fetch() requires a spider gets it passed in.""" + engine._downloader_fetch_needs_spider = True + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, request) + assert result == response + engine.downloader.fetch.assert_called_once_with(request, engine.spider) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestEngineDownload(TestEngineDownloadAsync): @@ -640,6 +655,146 @@ async def test_request_scheduled_signal(): crawler.signals.disconnect(signal_handler, signals.request_scheduled) +class TestEngineThrottling: + @pytest.fixture + def engine(self): + crawler = get_crawler(MySpider) + engine = ExecutionEngine(crawler, lambda _: None) + yield engine + engine.downloader.close() + + def test_pause_cancels_throttling_wakeup(self, engine): + wakeup = Mock() + engine._throttling_wakeup = wakeup + engine.pause() + assert engine.paused is True + wakeup.cancel.assert_called_once_with() + assert engine._throttling_wakeup is None + engine.unpause() + assert engine.paused is False + + def test_maybe_arm_throttling_wakeup_arms_timer(self, engine): + scheduler = Mock() + scheduler.has_pending_requests.return_value = True + scheduler.next_request_delay.return_value = 5.0 + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine._maybe_arm_throttling_wakeup() + assert engine._throttling_wakeup is not None + # Cancel the scheduled reactor call so it does not leak into other tests. + engine._cancel_throttling_wakeup() + + def test_maybe_arm_throttling_wakeup_no_delay(self, engine): + scheduler = Mock() + scheduler.has_pending_requests.return_value = True + scheduler.next_request_delay.return_value = None + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine._maybe_arm_throttling_wakeup() + assert engine._throttling_wakeup is None + + def test_warn_delayed_requests(self, engine): + engine._delayed_requests_warn_threshold = 1 + engine._throttling_waiting = {Request("http://a.example")} + engine._slot = Mock() + # A scheduler without next_request_delay is not throttling-aware, so the + # warning recommends switching to one. + engine._slot.scheduler = Mock(spec=BaseScheduler) + with LogCapture() as log: + engine._maybe_warn_delayed_requests() + # A second call is a no-op (the warning is emitted only once). + engine._maybe_warn_delayed_requests() + assert engine._delayed_requests_warned is True + log_text = str(log) + assert "requests held back by throttling" in log_text + assert log_text.count("ThrottlingAwareScheduler") == 1 + + def test_warn_delayed_requests_throttling_aware(self, engine): + engine._delayed_requests_warn_threshold = 1 + engine._throttling_waiting = {Request("http://a.example")} + engine._slot = Mock() + # A throttling-aware scheduler (one with next_request_delay) holds + # throttled requests itself, so no switch is recommended. + engine._slot.scheduler = Mock() + with LogCapture() as log: + engine._maybe_warn_delayed_requests() + assert "ThrottlingAwareScheduler" not in str(log) + + def test_spider_is_idle_false_while_scheduling(self, engine): + engine._slot = Mock() + engine.scraper.slot = Mock() + engine.scraper.slot.is_idle.return_value = True + engine.downloader = Mock() + engine.downloader.active = [] + engine._throttling_waiting = set() + engine._start = None + engine._scheduling = 1 + # An in-flight async enqueue keeps the spider from being considered idle. + assert engine.spider_is_idle() is False + + @coroutine_test + async def test_enqueue_request_async_dropped(self, engine): + scheduler = Mock() + + async def enqueue_request_async(request): + return False + + scheduler.enqueue_request_async = enqueue_request_async + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine.spider = Mock() + dropped = [] + + def on_dropped(request, spider): + dropped.append(request) + + engine.signals.connect(on_dropped, signals.request_dropped, weak=False) + engine._scheduling = 1 + request = Request("http://a.example") + await engine._enqueue_request_async(request) + assert dropped == [request] + assert engine._scheduling == 0 + engine._slot.nextcall.schedule.assert_called_once_with() + + @coroutine_test + async def test_enqueue_request_async_error(self, engine): + scheduler = Mock() + + async def enqueue_request_async(request): + raise RuntimeError("boom") + + scheduler.enqueue_request_async = enqueue_request_async + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine.spider = Mock() + engine._scheduling = 1 + with LogCapture() as log: + await engine._enqueue_request_async(Request("http://a.example")) + assert "Error while enqueuing request" in str(log) + assert engine._scheduling == 0 + engine._slot.nextcall.schedule.assert_called_once_with() + + @coroutine_test + async def test_enqueue_request_async_slot_gone(self, engine): + scheduler = Mock() + + async def enqueue_request_async(request): + # The spider is closed while the enqueue is in flight. + engine._slot = None + return True + + scheduler.enqueue_request_async = enqueue_request_async + slot = Mock() + slot.scheduler = scheduler + engine._slot = slot + engine.spider = Mock() + engine._scheduling = 1 + await engine._enqueue_request_async(Request("http://a.example")) + assert engine._scheduling == 0 + # No reschedule is attempted once the slot is gone. + slot.nextcall.schedule.assert_not_called() + + class TestEngineCloseSpider: """Tests for exception handling coverage during close_spider_async().""" diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 27c49ae41..92dfcafe5 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -448,6 +448,43 @@ class TestThrottlingAwarePriorityQueue: queue.pqueues[frozenset({"a.com"})] = queue.pqfactory(frozenset({"a.com"})) assert queue.next_request_delay() is None + @coroutine_test + async def test_next_request_delay_keeps_minimum(self): + crawler = get_crawler( + Spider, + settings_dict={ + "THROTTLING_SCOPES": { + "a.com": {"delay": 10.0}, + "b.com": {"delay": 1000.0}, + }, + "RANDOMIZE_DOWNLOAD_DELAY": False, + }, + ) + queue = self._queue(crawler) + # Two requests per scope so a blocked head remains after the first one + # (sendable, since no delay has accrued yet) is popped and reserved. + await self._push(queue, crawler, Request("http://a.com/1")) + await self._push(queue, crawler, Request("http://a.com/2")) + await self._push(queue, crawler, Request("http://b.com/1")) + await self._push(queue, crawler, Request("http://b.com/2")) + queue.pop() + queue.pop() + # Both scopes are now time-blocked; the smaller per-scope delay wins, + # so the larger one exercises the "not below the running minimum" branch. + delay = queue.next_request_delay() + assert delay == pytest.approx(10.0, abs=1.0) + + @coroutine_test + async def test_pop_handles_drained_selected_queue(self): + crawler = get_crawler(Spider) + queue = self._queue(crawler) + await self._push(queue, crawler, Request("http://a.com/1")) + inner = next(iter(queue.pqueues.values())) + # peek() still reports a sendable head, but pop() yields nothing: the + # request-is-None guard must not try to reserve a missing request. + inner.pop = lambda: None + assert queue.pop() is None + @coroutine_test async def test_contains(self): crawler = get_crawler(Spider) diff --git a/tests/test_throttling.py b/tests/test_throttling.py index e9fa15c63..6ee4a49df 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -478,6 +478,16 @@ class TestThrottlingScopeManager: scope.record_done(now=0.0) assert not event.called + def test_fire_slot_waiters_skips_already_fired(self): + scope = _scope_manager(config={"id": "x", "concurrency": 1}) + scope.record_sent(now=0.0) + event = scope.slot_event() + event.callback(None) # fired out-of-band before the slot frees up + # record_done() fires the waiters; the already-fired one is skipped + # rather than called a second time (which would raise). + scope.record_done(now=0.0) + assert event.called + def test_set_concurrency_respects_min(self): scope = _scope_manager(config={"id": "x", "min_concurrency": 3}) scope.set_concurrency(1) @@ -781,6 +791,42 @@ class TestThrottlingManagerEdges: # A second call is a no-op (the request was already delayed). await manager._apply_request_delay(request) + @coroutine_test + async def test_apply_request_delay_without_debug(self): + # Same as above but with debug logging off, so the delay is applied + # without emitting the debug message. + manager = _manager() + request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) + await manager._apply_request_delay(request) + assert request.meta["_throttling_delayed"] is True + + @coroutine_test + async def test_wait_for_slot_discards_unfired_events(self): + from scrapy.utils.asyncio import call_later # noqa: PLC0415 + + manager = _manager() + m1 = _scope_manager(config={"id": "a", "concurrency": 1}) + m2 = _scope_manager(config={"id": "b", "concurrency": 1}) + m1.record_sent(now=0.0) + m2.record_sent(now=0.0) + # Free m1's slot on the next tick so the wait wakes up with m1's event + # fired while m2's event is still pending. + call_later(0, m1.record_done) + await manager._wait_for_slot([m1, m2]) + # The still-pending m2 event is discarded from its waiter list. + assert m2._slot_waiters == [] + + def test_scope_manager_protocol_defaults(self): + from scrapy.throttling import ThrottlingScopeManagerProtocol # noqa: PLC0415 + + class _Concrete(ThrottlingScopeManagerProtocol): + pass + + crawler = get_crawler() + # The protocol's default from_crawler()/__init__ are usable as-is. + instance = _Concrete.from_crawler(crawler, {"id": "example.com"}) + assert isinstance(instance, _Concrete) + @coroutine_test async def test_process_exception_applies_backoff(self): manager = _manager() diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 76f7f95c7..f73d6ffd4 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -173,3 +173,20 @@ class TestWaitForFirst: done, pending = await wait_for_first([fired], timeout=1) assert done == {fired} assert pending == set() + + @coroutine_test + async def test_no_timeout(self): + # timeout=None -> no timeout deferred is created. + fired: Deferred[None] = Deferred() + fired.callback(None) + done, pending = await wait_for_first([fired]) + assert done == {fired} + assert pending == set() + + @coroutine_test + async def test_timeout_elapses(self): + # When the timeout fires first, every input deferred is still pending. + never: Deferred[None] = Deferred() + done, pending = await wait_for_first([never], timeout=0.01) + assert done == set() + assert pending == {never} From ee8c5f7aa4486abf9dd7a2e225140745055e40ea Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 16:00:11 +0200 Subject: [PATCH 16/55] Fix two environment-specific test failures --- scrapy/throttling.py | 6 ++---- tests/test_engine.py | 1 + tests/test_throttling.py | 11 ----------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index b46e192fa..f90a25dd8 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -800,11 +800,9 @@ class ThrottlingScopeManagerProtocol(Protocol): """ @classmethod - def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self: - return cls(crawler, config) + def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self: ... - def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: - pass + def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None: ... def can_send(self, now: float | None = None, amount: float | None = None) -> float: """Return the number of seconds to wait before a request for this scope diff --git a/tests/test_engine.py b/tests/test_engine.py index 148a7f4e7..60d911009 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -673,6 +673,7 @@ class TestEngineThrottling: engine.unpause() assert engine.paused is False + @pytest.mark.requires_reactor # call_later() needs a reactor or asyncio loop def test_maybe_arm_throttling_wakeup_arms_timer(self, engine): scheduler = Mock() scheduler.has_pending_requests.return_value = True diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 6ee4a49df..62efe29bb 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -816,17 +816,6 @@ class TestThrottlingManagerEdges: # The still-pending m2 event is discarded from its waiter list. assert m2._slot_waiters == [] - def test_scope_manager_protocol_defaults(self): - from scrapy.throttling import ThrottlingScopeManagerProtocol # noqa: PLC0415 - - class _Concrete(ThrottlingScopeManagerProtocol): - pass - - crawler = get_crawler() - # The protocol's default from_crawler()/__init__ are usable as-is. - instance = _Concrete.from_crawler(crawler, {"id": "example.com"}) - assert isinstance(instance, _Concrete) - @coroutine_test async def test_process_exception_applies_backoff(self): manager = _manager() From 5aae8334a5ddddc0df79dd33c974d0580108ce9b Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 19:00:06 +0200 Subject: [PATCH 17/55] Improve request and scope delay support --- docs/topics/throttling.rst | 55 ++++++++++++++++++++- scrapy/pqueues.py | 71 +++++++++++++++++++++++++-- scrapy/throttling.py | 98 +++++++++++++++++++++++++++++++++----- tests/test_pqueues.py | 92 +++++++++++++++++++++++++++++++++++ tests/test_scheduler.py | 27 +++++++++++ tests/test_throttling.py | 79 ++++++++++++++++++++++++++++-- 6 files changed, 401 insertions(+), 21 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 332a874c5..8d3ae004e 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -290,7 +290,6 @@ minimum delays (capped at :setting:`BACKOFF_MAX_DELAY`). .. seealso:: :setting:`REDIRECT_MAX_DELAY` - .. _crawl-delay: robots.txt @@ -313,6 +312,47 @@ If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it will be respected, but a warning will be logged about the discrepancy with ``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. +.. _delay-scope: + +Delaying a scope programmatically +================================= + +You can delay a :ref:`throttling scope ` on demand through +:meth:`crawler.throttler.delay_scope() +`: + +.. code-block:: python + + crawler.throttler.delay_scope("example.com", 30.0) + +This holds back every request of the scope for at least the given number of +seconds, counted as a :ref:`backoff ` trigger. + +It is useful to react to situations that :ref:`automatic backoff ` +cannot detect on its own, such as a soft block that comes back as a ``200`` +response. For example, a spider callback can slow down the whole domain when it +detects a maintenance page, and reschedule the current request: + +.. code-block:: python + + from scrapy import Request, Spider + from scrapy.utils.httpobj import urlparse_cached + + + class MySpider(Spider): + name = "myspider" + start_urls = ["https://example.com/"] + + def parse(self, response): + if "under maintenance" in response.text: + scope = urlparse_cached(response).netloc + self.crawler.throttler.delay_scope(scope, 600.0) + yield response.request.replace(dont_filter=True) + return + # Normal parsing follows. + +Unlike :ref:`untrusted delays `, this delay is **not** +capped at :setting:`BACKOFF_MAX_DELAY`. .. _per-request-throttling: @@ -374,6 +414,19 @@ regardless of its scopes, set the ``throttling_delay`` request metadata key: The delay is applied once, the first time the request reaches the throttling gate. +``throttling_delay`` defines only the *earliest* time the request may be sent, +not the exact time: once the delay elapses, the request still competes with +every other pending request for its scopes. If you want it sent **as soon as** +its delay elapses, give it a higher :attr:`~scrapy.Request.priority` too: + +.. code-block:: python + + Request("https://example.com/slow", meta={"throttling_delay": 5.0}, priority=1) + +Without a higher priority, a backlog of requests ahead of it in a FIFO queue +could keep it waiting well past the configured delay; a higher priority puts it +at the front of the queue, so it goes out right after its delay. + .. reqmeta:: throttling_dont_track Excluding a request from throttling state diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 60675fe7f..75dc757db 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,8 +1,10 @@ from __future__ import annotations import hashlib +import heapq import json import logging +import time from typing import TYPE_CHECKING, Protocol, cast from scrapy.utils.misc import build_from_crawler @@ -455,9 +457,9 @@ def _scope_set_from_key(key: str) -> frozenset[ScopeID]: class ThrottlingAwarePriorityQueue: - """Priority queue that partitions requests by their full :ref:`throttling - scope set ` and only ever pops a request that can be - sent right now. + """Priority queue that only ever pops a request that can be sent right now + based on its :ref:`throttling scope set ` and + per-request :reqmeta:`throttling_delay`. The downstream queue class must support ``peek``. @@ -520,6 +522,11 @@ class ThrottlingAwarePriorityQueue: # scope set -> priority queue self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {} + # Min-heap of (deadline, seq, scope_set, request) for requests held back + # by a per-request throttling_delay; seq keeps ordering stable and + # avoids comparing requests when deadlines tie. + self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] + self._delayed_seq: int = 0 if slot_startprios: for set_key, startprios in slot_startprios.items(): scope_set = _scope_set_from_key(set_key) @@ -537,10 +544,52 @@ class ThrottlingAwarePriorityQueue: ) def push(self, request: Request, scope_set: frozenset[ScopeID]) -> None: + now = time.monotonic() + self._promote_ready(now) + delay = self._throttler.get_request_delay(request, now) + if delay > 0: + self._delayed_seq += 1 + heapq.heappush( + self._delayed, (now + delay, self._delayed_seq, scope_set, request) + ) + return + self._push_to_queue(request, scope_set) + + def _push_to_queue(self, request: Request, scope_set: frozenset[ScopeID]) -> None: if scope_set not in self.pqueues: self.pqueues[scope_set] = self.pqfactory(scope_set) self.pqueues[scope_set].push(request) + def _promote_ready(self, now: float) -> None: + """Move every held-back request whose per-request delay has elapsed into + its scope-set queue, where it competes normally for its scopes.""" + while self._delayed and self._delayed[0][0] <= now: + self._release_delayed(heapq.heappop(self._delayed)) + + def _release_delayed( + self, entry: tuple[float, int, frozenset[ScopeID], Request] + ) -> None: + _, _, scope_set, request = entry + # The per-request delay has been honored (or the queue is closing), so + # mark it consumed: the request must not be delayed again, and on resume + # it must not re-block its scope set on a stale, no-longer-meaningful + # deadline. + request.meta["_throttling_delayed"] = True + try: + self._push_to_queue(request, scope_set) + except ValueError as e: + # A disk queue serializes on push; held-back requests defer that + # serialization until here, so a non-serializable one would + # otherwise raise while flushing on close and take the rest of the + # disk queue down with it. Drop it with a warning instead, matching + # how the scheduler handles unserializable requests at enqueue time. + logger.warning( + "Unable to serialize request: %(request)s - reason: %(reason)s", + {"request": request, "reason": e}, + exc_info=True, + extra={"spider": getattr(self.crawler, "spider", None)}, + ) + def _select( self, ) -> tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None: @@ -553,6 +602,7 @@ class ThrottlingAwarePriorityQueue: :meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the scopes of the queue), i.e. by preferring the least-busy scopes. """ + self._promote_ready(time.monotonic()) best_sort_key: tuple[int, float] | None = None best: tuple[frozenset[ScopeID], ScrapyPriorityQueue] | None = None for scope_set, queue in self.pqueues.items(): @@ -588,6 +638,8 @@ class ThrottlingAwarePriorityQueue: return selected[1].peek() def next_request_delay(self) -> float | None: + now = time.monotonic() + self._promote_ready(now) delay: float | None = None for queue in self.pqueues.values(): head = queue.peek() @@ -600,9 +652,19 @@ class ThrottlingAwarePriorityQueue: continue if delay is None or head_delay < delay: delay = head_delay + # A request held back only by its own throttling_delay is not in any + # scope-set queue, so factor in when the earliest one is due. + if self._delayed: + next_delayed = max(0.0, self._delayed[0][0] - now) + if delay is None or next_delayed < delay: + delay = next_delayed return delay def close(self) -> dict[str, list[int]]: + # Flush held-back requests into their scope-set queues so they are + # persisted (and restored on resume) rather than lost. + while self._delayed: + self._release_delayed(heapq.heappop(self._delayed)) active = { _scope_set_key(scope_set): queue.close() for scope_set, queue in self.pqueues.items() @@ -611,7 +673,8 @@ class ThrottlingAwarePriorityQueue: return active def __len__(self) -> int: - return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + queued = sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + return queued + len(self._delayed) def __contains__(self, scope_set: frozenset[ScopeID]) -> bool: return scope_set in self.pqueues diff --git a/scrapy/throttling.py b/scrapy/throttling.py index f90a25dd8..e603f1fbe 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -323,10 +323,10 @@ class ThrottlingManagerProtocol(Protocol): *and* a concurrency slot is free in every scope. This is the synchronous, non-blocking counterpart of :meth:`acquire`, - used by a :ref:`throttling-aware scheduler ` - to decide whether a request can be dequeued now. It assumes the scopes - of *request* have already been resolved (e.g. by an earlier - :meth:`get_scopes` call at enqueue time). + used by a :ref:`throttling-aware scheduler + ` to decide whether a request can be + dequeued now. It assumes the scopes of *request* have already been + resolved (e.g. by an earlier :meth:`get_scopes` call at enqueue time). """ def reserve(self, request: Request) -> None: @@ -359,6 +359,31 @@ class ThrottlingManagerProtocol(Protocol): preferring the least-loaded ones. """ + def get_request_delay(self, request: Request, now: float | None = None) -> float: + """Return how many seconds *request* must still be held individually + because of its :reqmeta:`throttling_delay`, or ``0.0`` if it has none + or it has already elapsed. The one-time delay is started on the first + call. + + Unlike a scope delay, this affects only *request*: a + :ref:`throttling-aware scheduler ` must + hold the request back on its own, **without** blocking other requests + that share its scopes. + """ + + def delay_scope(self, scope_id: str, delay: float) -> None: + """Hold back every request of the scope identified by *scope_id* for at + least *delay* seconds, counted as a :ref:`backoff ` trigger + for the scope. + + This is the programmatic equivalent of a :ref:`Retry-After + ` response header, available to any component through + :attr:`crawler.throttler `. Unlike + those headers, *delay* is **not** capped at + :setting:`BACKOFF_MAX_DELAY`: that cap guards against untrusted input, + whereas a ``delay_scope`` call is trusted. + """ + async def process_response(self, response: Response) -> None: """Update the throttling state based on *response*.""" @@ -551,7 +576,7 @@ class ThrottlingManager: return now = time.monotonic() self._maybe_evict(now) - await self._apply_request_delay(request) + await self._delay_request(request) scope_values = list(iter_scope_values(await self.get_scopes(request))) if not scope_values: return @@ -599,6 +624,8 @@ class ThrottlingManager: def is_ready(self, request: Request) -> bool: now = time.monotonic() + if self._request_delay_deadline(request, now) > now: + return False for scope_id, value in self._cached_scope_values(request): manager = self._get_scope_manager(scope_id) if manager.can_send(now=now, amount=value) > 0: @@ -618,7 +645,7 @@ class ThrottlingManager: def time_until_ready(self, request: Request) -> float | None: now = time.monotonic() - wait = 0.0 + wait = max(0.0, self._request_delay_deadline(request, now) - now) for scope_id, value in self._cached_scope_values(request): manager = self._get_scope_manager(scope_id) wait = max(wait, manager.can_send(now=now, amount=value)) @@ -633,6 +660,10 @@ class ThrottlingManager: return 0.0 return active / limit + def get_request_delay(self, request: Request, now: float | None = None) -> float: + now = time.monotonic() if now is None else now + return max(0.0, self._request_delay_deadline(request, now) - now) + async def _wait_for_slot(self, managers: list[Any]) -> None: """Block until any of *managers* frees a concurrency slot. @@ -649,7 +680,7 @@ class ThrottlingManager: if event in pending: manager.discard_slot_event(event) - async def _apply_request_delay(self, request: Request) -> None: + async def _delay_request(self, request: Request) -> None: """Honor the :reqmeta:`throttling_delay` meta key by holding *request* for the requested number of seconds the first time it is processed.""" delay = request.meta.get("throttling_delay") @@ -660,6 +691,31 @@ class ThrottlingManager: logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") await sleep(float(delay)) + def _request_delay_deadline(self, request: Request, now: float) -> float: + """Return the monotonic time before which *request* must not be sent due + to its :reqmeta:`throttling_delay`, or ``0.0`` if it has none. + + This is the readiness-API counterpart of :meth:`_delay_request`: + a throttling-aware scheduler gates requests through :meth:`is_ready` and + :meth:`time_until_ready` instead of awaiting :meth:`acquire`, so the + delay is enforced by holding back the request until this deadline rather + than by sleeping. The deadline is computed once, the first time the + request reaches the gate, and stored so later polls reuse it. + + A request whose delay has already been honored (the ``_throttling_delayed`` + flag, also set by :meth:`_delay_request`) is never delayed again, + which keeps a resumed crawl from re-blocking on a stale deadline.""" + delay = request.meta.get("throttling_delay") + if not delay or request.meta.get("_throttling_delayed"): + return 0.0 + deadline = request.meta.get("_throttling_delay_deadline") + if deadline is None: + deadline = now + float(delay) + request.meta["_throttling_delay_deadline"] = deadline + if self._debug: + logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") + return deadline + async def process_response(self, response: Response) -> None: data = await self.get_response_backoff(response) self._apply_backoff(data) @@ -752,6 +808,13 @@ class ThrottlingManager: manager.set_base_delay(capped) manager.set_concurrency(1) + def delay_scope(self, scope_id: ScopeID, delay: float) -> None: + if self._debug: + logger.debug(f"Delaying scope {scope_id} for {delay:.2f}s") + # Like a Retry-After / RateLimit-Reset header, this is a hard minimum + # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. + self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) + def _maybe_evict(self, now: float) -> None: if self._max_idle <= 0: return @@ -823,13 +886,21 @@ class ThrottlingScopeManagerProtocol(Protocol): downloading, freeing its concurrency slot.""" def record_backoff( - self, delay: float | None = None, now: float | None = None + self, + delay: float | None = None, + now: float | None = None, + cap: bool = True, ) -> None: """Apply a backoff to this scope. *delay*, when given, is a hard minimum delay in seconds (e.g. from a ``Retry-After`` header). When omitted, an exponential backoff step is applied instead. + + *cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`. It is ``True`` for + untrusted input such as response headers, and may be set to ``False`` + for trusted, programmatic delays (see + :meth:`ThrottlingManagerProtocol.delay_scope`). """ def reconcile_quota( @@ -1128,7 +1199,10 @@ class ThrottlingScopeManager: event.callback(None) def record_backoff( - self, delay: float | None = None, now: float | None = None + self, + delay: float | None = None, + now: float | None = None, + cap: bool = True, ) -> None: now = self._now(now) self._last_seen = now @@ -1136,9 +1210,11 @@ class ThrottlingScopeManager: self._backoff_level += 1 self._rampup_backoffs += 1 if delay is not None: - hard = min(float(delay), self._max_delay) + hard = min(float(delay), self._max_delay) if cap else float(delay) self._in_backoff_until = now + hard - self._delay = min(max(self._delay, hard, self._min_delay), self._max_delay) + self._delay = max(self._delay, hard, self._min_delay) + if cap: + self._delay = min(self._delay, self._max_delay) else: grown = ( self._delay * self._delay_factor if self._delay > 0 else self._min_delay diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 92dfcafe5..5fc4207e4 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -368,6 +368,98 @@ class TestThrottlingAwarePriorityQueue: assert delay is not None assert delay == pytest.approx(1000.0, abs=1.0) + @coroutine_test + async def test_pop_holds_request_with_throttling_delay(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + await self._push( + queue, + crawler, + Request("http://slow.com/1", meta={"throttling_delay": 1000.0}), + ) + await self._push(queue, crawler, Request("http://fast.com/1")) + # The delayed request is held back even though its scope is otherwise + # unconstrained; the request without a delay is served. + popped = [queue.pop(), queue.pop()] + urls = [r.url if r else None for r in popped] + assert "http://fast.com/1" in urls + assert None in urls + assert len(queue) == 1 + delay = queue.next_request_delay() + assert delay is not None + assert delay == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_delayed_request_does_not_block_scope_set(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + # Both requests share the same (example.com) scope set; only the first + # carries a per-request delay. + await self._push( + queue, + crawler, + Request("http://example.com/slow", meta={"throttling_delay": 1000.0}), + ) + await self._push(queue, crawler, Request("http://example.com/fast")) + # The delayed request is held aside, so the other request in the same + # scope set is served right away instead of being stuck behind it. + assert queue.pop().url == "http://example.com/fast" + # The delayed request is not lost, just not poppable yet. + assert queue.pop() is None + assert len(queue) == 1 + assert queue.next_request_delay() == pytest.approx(1000.0, abs=1.0) + + @coroutine_test + async def test_delayed_request_promoted_when_due(self): + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + queue = self._queue(crawler) + request = Request("http://example.com/slow", meta={"throttling_delay": 1000.0}) + await self._push(queue, crawler, request) + assert queue.pop() is None # held back by its per-request delay + # Once the delay elapses the request is promoted into its scope-set + # queue, served, and flagged so the delay is not applied a second time. + queue._promote_ready(queue._delayed[0][0]) + popped = queue.pop() + assert popped is request + assert popped.meta["_throttling_delayed"] is True + assert len(queue) == 0 + + @coroutine_test + async def test_delayed_request_persisted_on_close(self): + # With a JOBDIR (disk queue), a request held back by its per-request + # delay must not be lost on a graceful stop: close() flushes it to disk + # so it is restored on resume. + crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) + temp_dir = tempfile.mkdtemp() + queue = build_from_crawler( + ThrottlingAwarePriorityQueue, + crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=temp_dir, + ) + await self._push( + queue, + crawler, + Request("http://example.com/slow", meta={"throttling_delay": 1000.0}), + ) + assert len(queue) == 1 # held in memory, not yet in any scope-set queue + state = queue.close() # graceful stop + + resumed = build_from_crawler( + ThrottlingAwarePriorityQueue, + crawler, + downstream_queue_cls=PickleFifoDiskQueue, + key=temp_dir, + startprios=state, + ) + assert len(resumed) == 1 + popped = resumed.pop() + assert popped is not None + assert popped.url == "http://example.com/slow" + # Its delay is marked consumed, so it does not re-block on resume. + assert popped.meta["_throttling_delayed"] is True + resumed.close() + @coroutine_test async def test_least_loaded_first(self): crawler = get_crawler( diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ad91efcc7..12dc61461 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -515,6 +515,33 @@ class TestThrottlingAwareScheduler: assert scheduler.next_request_delay() is None scheduler.close("finished") + @coroutine_test + async def test_delayed_request_survives_jobdir_stop(self, tmp_path: Path) -> None: + # A request held back by its per-request throttling_delay must not be + # lost on a graceful stop when a JOBDIR is configured: it is flushed to + # the disk queue on close and restored on resume. + crawler = self._crawler( + {"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + scheduler = self._scheduler(crawler) + request = Request("http://a.com/slow", meta={"throttling_delay": 1000.0}) + assert await scheduler.enqueue_request_async(request) is True + assert len(scheduler) == 1 + # The delay holds it back, so nothing is dequeued before the stop. + assert scheduler.next_request() is None + scheduler.close("finished") + + # Resume from the same JOBDIR: the request is still there and, having + # been held once, is now sendable. + resumed = self._scheduler( + self._crawler({"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False}) + ) + assert len(resumed) == 1 + resumed_request = resumed.next_request() + assert resumed_request is not None + assert resumed_request.url == "http://a.com/slow" + resumed.close("finished") + @coroutine_test async def test_enqueue_async_filters_duplicates(self) -> None: crawler = self._crawler( diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 62efe29bb..3dc7727f9 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -381,6 +381,12 @@ class TestThrottlingScopeManager: scope.record_backoff(delay=backoff_delay, now=0.0) assert scope.can_send(now=0.0) == pytest.approx(expected) + def test_uncapped_backoff_delay(self): + # cap=False (used by trusted delay_scope() calls) ignores BACKOFF_MAX_DELAY. + scope = _scope_manager({"BACKOFF_MAX_DELAY": 10.0}) + scope.record_backoff(delay=999.0, now=0.0, cap=False) + assert scope.can_send(now=0.0) == pytest.approx(999.0) + def test_recovery_after_window(self): scope = _scope_manager( { @@ -598,6 +604,69 @@ class TestThrottlingManagerReadiness: assert manager.is_ready(second) is False assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0) + @coroutine_test + async def test_throttling_delay_blocks_until_deadline(self): + manager = _manager({"THROTTLING_DEBUG": True}) + request = Request("http://example.com/a", meta={"throttling_delay": 100.0}) + await manager.get_scopes(request) + # The per-request delay holds back the request even though its scope is + # otherwise unconstrained. + assert manager.is_ready(request) is False + assert manager.time_until_ready(request) == pytest.approx(100.0, abs=1.0) + # The deadline is computed once and reused by later polls. + deadline = request.meta["_throttling_delay_deadline"] + assert manager.is_ready(request) is False + assert request.meta["_throttling_delay_deadline"] == deadline + + @coroutine_test + async def test_throttling_delay_not_reapplied_once_consumed(self): + # A request whose delay was already honored (e.g. promoted out of a + # throttling-aware queue's holding area, or restored on resume) is + # ready, so it cannot re-block its scope set on a stale deadline. + manager = _manager() + request = Request( + "http://example.com/a", + meta={"throttling_delay": 100.0, "_throttling_delayed": True}, + ) + await manager.get_scopes(request) + assert manager.is_ready(request) is True + assert manager.get_request_delay(request) == 0.0 + + @coroutine_test + async def test_get_request_delay(self): + manager = _manager() + assert manager.get_request_delay( + Request("http://example.com/a", meta={"throttling_delay": 100.0}) + ) == pytest.approx(100.0, abs=1.0) + # A request without a per-request delay is not held individually. + assert manager.get_request_delay(Request("http://example.com/b")) == 0.0 + + @coroutine_test + async def test_delay_scope(self): + manager = _manager( + {"THROTTLING_DEBUG": True, "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + assert manager.is_ready(request) is True + # A component can delay a whole scope on demand, like a Retry-After + # response header does. + manager.delay_scope("example.com", 50.0) + assert manager.is_ready(request) is False + assert manager.time_until_ready(request) == pytest.approx(50.0, abs=1.0) + + @coroutine_test + async def test_delay_scope_bypasses_max_delay(self): + # BACKOFF_MAX_DELAY caps untrusted input (headers), but delay_scope is a + # trusted call, so it may exceed the cap. + manager = _manager( + {"BACKOFF_MAX_DELAY": 30.0, "RANDOMIZE_DOWNLOAD_DELAY": False} + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + manager.delay_scope("example.com", 1000.0) + assert manager.time_until_ready(request) == pytest.approx(1000.0, abs=1.0) + @coroutine_test async def test_reserve_blocks_on_concurrency(self): manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) @@ -783,21 +852,21 @@ class TestThrottlingManagerEdges: assert r2 in manager._reserved @coroutine_test - async def test_apply_request_delay(self): + async def test_delay_request(self): manager = _manager({"THROTTLING_DEBUG": True}) request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) - await manager._apply_request_delay(request) + await manager._delay_request(request) assert request.meta["_throttling_delayed"] is True # A second call is a no-op (the request was already delayed). - await manager._apply_request_delay(request) + await manager._delay_request(request) @coroutine_test - async def test_apply_request_delay_without_debug(self): + async def test_delay_request_without_debug(self): # Same as above but with debug logging off, so the delay is applied # without emitting the debug message. manager = _manager() request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) - await manager._apply_request_delay(request) + await manager._delay_request(request) assert request.meta["_throttling_delayed"] is True @coroutine_test From d23c44858af4cc33b9fd2753cfa8de7f9ff65c56 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 21:41:45 +0200 Subject: [PATCH 18/55] Address issues --- scrapy/throttling.py | 147 +++++++++++++++++++++++++++++---------- tests/test_throttling.py | 123 +++++++++++++++++++++++++++++++- 2 files changed, 230 insertions(+), 40 deletions(-) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index e603f1fbe..bdddd3d3b 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -396,15 +396,28 @@ _GetScopesMethod = TypeVar( ) -def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: - """Decorator to cache the result of - :meth:`~ThrottlingManagerProtocol.get_scopes` calls. +# Request.meta key under which scope_cache persists the resolved scopes so that +# they survive a request being serialized to and restored from a disk queue. +_RESOLVED_SCOPES_META_KEY = "_throttling_resolved_scopes" - It should be used so that calls to - :meth:`~ThrottlingManagerProtocol.get_scopes` from methods like - :meth:`~ThrottlingManagerProtocol.get_response_backoff` or - :meth:`~ThrottlingManagerProtocol.get_exception_backoff` do not become - unnecessarily expensive. + +def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: + """Decorator for :meth:`~ThrottlingManagerProtocol.get_scopes` + implementations that persists the resolved scopes on ``request.meta``. + + The readers of the resolved scopes — the synchronous readiness API of a + :ref:`throttling-aware scheduler ` and the + backoff methods — read this persisted value instead of resolving the scopes + again, so they stay cheap and consistent, and it survives a request being + serialized to and restored from a :ref:`disk queue ` (which is + what lets the readiness API resolve the scopes of a restored request + synchronously). + + The decorated method always re-resolves; it never reads the persisted value + back. So a request that inherited ``request.meta`` from another one (e.g. a + redirect built with :meth:`Request.replace() `, + which copies ``meta``) resolves its own scopes and overwrites the inherited + ones rather than reusing them. For example: @@ -422,15 +435,12 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: @wraps(f) async def wrapper(self: Any, request: Request) -> RequestScopes: - cache: WeakKeyDictionary[Request, RequestScopes] | None = self.__dict__.get( - "_scope_cache" - ) - if cache is None: - cache = self.__dict__["_scope_cache"] = WeakKeyDictionary() - if request in cache: - return cache[request] scopes = await f(self, request) - cache[request] = scopes + # Materialize one-shot iterables so the persisted value stays + # re-iterable and serializable. + if not isinstance(scopes, (str, dict)) and isinstance(scopes, Iterable): + scopes = list(scopes) + request.meta[_RESOLVED_SCOPES_META_KEY] = scopes return scopes return wrapper # type: ignore[return-value] @@ -476,7 +486,6 @@ class ThrottlingManager: "THROTTLING_SCOPES" ) self._scope_managers: dict[ScopeID, ThrottlingScopeManagerProtocol] = {} - self._global_concurrency: int = crawler.settings.getint("CONCURRENT_REQUESTS") self._last_eviction: float | None = None # Concurrency slots reserved by acquire(), to be released once the # request finishes downloading. @@ -491,12 +500,13 @@ class ThrottlingManager: def _resolve_scopes_sync(self, request: Request) -> RequestScopes: """Best-effort synchronous scope resolution. - It mirrors :meth:`get_scopes`, and is used by the synchronous + It backs :meth:`get_scopes` and is also the fallback for the synchronous readiness methods (:meth:`is_ready`, :meth:`reserve`, - :meth:`time_until_ready`) when the cached result of an earlier - :meth:`get_scopes` call is not available (e.g. for a request restored - from disk). Subclasses whose :meth:`get_scopes` cannot be resolved - synchronously should rely on the enqueue-time cache instead. + :meth:`time_until_ready`) when no scopes were persisted on + ``request.meta`` by an earlier :meth:`get_scopes` call (which normally + happens at enqueue time and survives disk restores; see + :func:`scope_cache`). Subclasses whose :meth:`get_scopes` cannot be + resolved synchronously rely on that persisted value instead. """ scopes = request.meta.get("throttling_scopes") if scopes is not None: @@ -507,16 +517,34 @@ class ThrottlingManager: self, request: Request ) -> list[tuple[ScopeID, float | None]]: """Return the ``(scope_id, quota_amount)`` pairs of *request*, reading - the cache populated by :meth:`get_scopes` and falling back to - :meth:`_resolve_scopes_sync`.""" - cache: WeakKeyDictionary[Request, RequestScopes] | None = self.__dict__.get( - "_scope_cache" - ) - scopes = cache.get(request) if cache is not None else None - if scopes is None: + the scopes persisted on ``request.meta`` by :meth:`get_scopes` (via + :func:`scope_cache`) and falling back to :meth:`_resolve_scopes_sync`. + + The persisted value is authoritative here: a request reaching the + readiness API has already been enqueued and filed under the queue for + these very scopes. + """ + if _RESOLVED_SCOPES_META_KEY in request.meta: + scopes = cast("RequestScopes", request.meta[_RESOLVED_SCOPES_META_KEY]) + else: scopes = self._resolve_scopes_sync(request) return list(iter_scope_values(scopes)) + async def _scopes(self, request: Request) -> RequestScopes: + """Return the scopes of *request*, reusing those persisted on + ``request.meta`` by an earlier :meth:`get_scopes` call (see + :func:`scope_cache`) and resolving them only as a fallback. + + Used by the backoff methods so that a request whose scopes were already + resolved (at enqueue time, or by :meth:`acquire`) is not resolved again + — which, for a request restored from a disk queue, would also risk + attributing the backoff to different scopes than the ones it was sent + under. + """ + if _RESOLVED_SCOPES_META_KEY in request.meta: + return cast("RequestScopes", request.meta[_RESOLVED_SCOPES_META_KEY]) + return await self.get_scopes(request) + async def get_initial_backoff(self) -> BackoffData: return None @@ -526,7 +554,7 @@ class ThrottlingManager: return None if response.status not in self._backoff_http_codes: return None - scopes = await self.get_scopes(response.request) + scopes = await self._scopes(response.request) if delay := self.get_response_delay(response): scopes = {scope: {"delay": delay} for scope in iter_scopes(scopes)} return scopes @@ -550,7 +578,7 @@ class ThrottlingManager: if request.meta.get("throttling_dont_track"): return None if isinstance(exception, self._backoff_exceptions): - return await self.get_scopes(request) + return await self._scopes(request) return None # -- Scope-state coordination (called from the request lifecycle) -------- @@ -635,6 +663,11 @@ class ThrottlingManager: return True def reserve(self, request: Request) -> None: + # A throttling-aware scheduler reserves every request before handing it + # to the engine, so acquire() always returns early for it and never + # gets to evict idle scopes; do it here so their managers do not pile + # up on broad crawls. + self._maybe_evict(time.monotonic()) managers = [ (self._get_scope_manager(scope_id), value) for scope_id, value in self._cached_scope_values(request) @@ -652,13 +685,7 @@ class ThrottlingManager: return wait if wait > 0 else None def scope_load(self, scope_id: ScopeID) -> float: - manager = self._get_scope_manager(scope_id) - active: int = getattr(manager, "_active", 0) - concurrency: int | None = getattr(manager, "_concurrency", None) - limit = concurrency if concurrency is not None else self._global_concurrency - if not limit: - return 0.0 - return active / limit + return self._get_scope_manager(scope_id).load() def get_request_delay(self, request: Request, now: float | None = None) -> float: now = time.monotonic() if now is None else now @@ -933,6 +960,19 @@ class ThrottlingScopeManagerProtocol(Protocol): Return ``False`` when no concurrency limit is enforced. """ + def load(self) -> float: + """Return the current load of this scope: a non-negative number, with + ``1.0`` meaning "as busy as its concurrency limit allows". + + A :ref:`throttling-aware scheduler ` uses + this to break ties between equally-prioritized requests, preferring the + least-loaded scopes. The reference implementation returns active sends + divided by the concurrency limit (falling back to + :setting:`CONCURRENT_REQUESTS` when the scope enforces no explicit + limit), but any consistent busyness metric works; return ``0.0`` when + none is meaningful. + """ + def slot_event(self) -> Deferred[None]: """Return a :class:`~twisted.internet.defer.Deferred` that fires when a concurrency slot next becomes available (e.g. when @@ -1042,6 +1082,9 @@ class ThrottlingScopeManager: self._concurrency = self._min_concurrency else: self._concurrency = None + # Used as the load denominator when the scope enforces no explicit + # concurrency limit (see load()). + self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") # Quota. quota = config.get("quota") @@ -1090,6 +1133,14 @@ class ThrottlingScopeManager: def _recover(self, now: float) -> None: if self._backoff_level == 0 or self._last_backoff_time is None: return + if self._window <= 0: + # A non-positive window has no recovery cadence to step through (and + # would spin forever on a zero-length step), so recover at once. + self._backoff_level = 0 + self._delay = self._base_delay + self._in_backoff_until = None + self._last_backoff_time = None + return while self._backoff_level > 0 and now - self._last_backoff_time >= self._window: self._backoff_level -= 1 self._last_backoff_time += self._window @@ -1105,6 +1156,10 @@ class ThrottlingScopeManager: under :setting:`RAMPUP_BACKOFF_TARGET` backoff triggers.""" if not self._rampup_enabled: return + if self._window <= 0: + # No window means no cadence to ramp up on (and a zero-length step + # would spin forever). + return if self._rampup_window_start is None: self._rampup_window_start = now return @@ -1131,6 +1186,12 @@ class ThrottlingScopeManager: def _maybe_reset_quota(self, now: float) -> None: if self._quota is None: return + if self._quota_window <= 0: + # A non-positive window has no reset cadence to step through (and + # would spin forever on a zero-length step), so keep it reset. + self._consumed = 0.0 + self._quota_window_start = now + return if self._quota_window_start is None: self._quota_window_start = now return @@ -1180,6 +1241,16 @@ class ThrottlingScopeManager: def concurrency_blocked(self) -> bool: return self._concurrency is not None and self._active >= self._concurrency + def load(self) -> float: + limit = ( + self._concurrency + if self._concurrency is not None + else self._global_concurrency + ) + if not limit: + return 0.0 + return self._active / limit + def slot_event(self) -> Deferred[None]: """Return a Deferred that fires when a concurrency slot next frees up (via :meth:`record_done`) or the limit is raised (via diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 3dc7727f9..9ba6d51eb 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -17,9 +17,11 @@ from scrapy.throttling import ( add_scope, iter_scope_values, iter_scopes, + scope_cache, update_scope_backoff, ) from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider from tests.utils.decorators import coroutine_test @@ -69,8 +71,8 @@ class TestThrottlingManager: manager = _manager() request = Request("http://example.com/a") first = await manager.get_scopes(request) - # A second call returns the cached value (same object identity for dicts, - # equal value for strings). + # get_scopes is deterministic per request, so a second call yields the + # same scopes. assert await manager.get_scopes(request) == first @coroutine_test @@ -87,6 +89,80 @@ class TestThrottlingManager: ) assert await manager.get_scopes(request) == {"api": 2.0} + @coroutine_test + async def test_get_scopes_persisted_in_meta(self): + from scrapy.throttling import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 + + manager = _manager() + request = Request("http://example.com/a") + scopes = await manager.get_scopes(request) + assert request.meta[_RESOLVED_SCOPES_META_KEY] == scopes + + @coroutine_test + async def test_scope_cache_works_without_crawler(self): + # scope_cache only persists to meta; it needs nothing from the manager. + from scrapy.throttling import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 + + class CrawlerlessManager: + @scope_cache + async def get_scopes(self, request): + return "scope" + + request = Request("http://example.com/a") + assert await CrawlerlessManager().get_scopes(request) == "scope" + assert request.meta[_RESOLVED_SCOPES_META_KEY] == "scope" + + @coroutine_test + async def test_backoff_reuses_persisted_scopes(self): + # Once get_scopes has resolved and persisted the scopes, the backoff + # path reuses them instead of resolving again. + calls = [] + + class CountingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + calls.append(request.url) + return urlparse_cached(request).netloc + + manager = CountingManager( + get_crawler(settings_dict={"BACKOFF_EXCEPTIONS": ["builtins.ValueError"]}) + ) + request = Request("http://example.com/a") + await manager.get_scopes(request) + assert calls == ["http://example.com/a"] + assert ( + await manager.get_exception_backoff(request, ValueError()) == "example.com" + ) + # No second resolution. + assert calls == ["http://example.com/a"] + + @coroutine_test + async def test_get_scopes_survives_disk_roundtrip(self): + from scrapy.utils.request import request_from_dict # noqa: PLC0415 + + manager = _manager() + request = Request( + "http://example.com/a", meta={"throttling_scopes": {"bucket": 3.0}} + ) + await manager.get_scopes(request) + # A request restored from a disk queue is a fresh object; the synchronous + # readiness path must still recover its scopes (with quota values) from + # the persisted meta, without re-running get_scopes. + restored = request_from_dict(request.to_dict()) + assert manager._cached_scope_values(restored) == [("bucket", 3.0)] + + @coroutine_test + async def test_get_scopes_reresolved_after_cross_host_replace(self): + # A redirect built with Request.replace() copies meta (including the + # persisted scopes), but get_scopes always re-resolves (it never reads + # the persisted value back), so it must not reuse the original host's + # scopes. + manager = _manager() + request = Request("http://example.com/a") + assert await manager.get_scopes(request) == "example.com" + redirected = request.replace(url="http://other.example/a") + assert await manager.get_scopes(redirected) == "other.example" + @coroutine_test async def test_get_initial_backoff_none(self): manager = _manager() @@ -324,6 +400,21 @@ class TestThrottlingManager: # even though it has been idle for longer than THROTTLING_SCOPE_MAX_IDLE. assert "example.com" in manager._scope_managers + def test_reserve_evicts_idle_scopes(self): + # A throttling-aware scheduler reserves every request before the engine + # reaches acquire() (which fast-paths reserved requests), so reserve() + # must be the hook that evicts idle scope managers; otherwise they pile + # up unbounded on broad crawls. + manager = _manager({"THROTTLING_SCOPE_MAX_IDLE": 1.0}) + idle = manager._get_scope_manager("idle.example") + # Make it look long-idle: a finished send in the distant monotonic past. + idle.record_sent(now=0.0) + idle.record_done(now=0.0) + assert "idle.example" in manager._scope_managers + manager.reserve(Request("http://active.example/1")) + assert "idle.example" not in manager._scope_managers + assert "active.example" in manager._scope_managers + class TestThrottlingScopeManager: def test_no_delay_by_default(self): @@ -467,6 +558,34 @@ class TestThrottlingScopeManager: scope.record_done(now=0.0) assert event.called + def test_zero_backoff_window_recovers_at_once(self): + # A non-positive window must not make _recover spin forever; it recovers + # fully on the next can_send() instead. + scope = _scope_manager(settings={"BACKOFF_WINDOW": 0}, config={"id": "x"}) + scope.record_backoff(now=0.0) + assert scope._backoff_level == 1 + scope.can_send(now=10.0) + assert scope._backoff_level == 0 + assert scope._delay == scope._base_delay + + def test_zero_quota_window_keeps_quota_reset(self): + # A non-positive quota window must not make _maybe_reset_quota spin + # forever; the quota stays continuously reset instead. + scope = _scope_manager(config={"id": "x", "quota": 10.0, "window": 0}) + scope.record_sent(now=0.0, amount=10.0) + assert scope.can_send(now=1.0, amount=5.0) == 0.0 + assert scope._consumed == 0.0 + + def test_zero_window_disables_rampup(self): + # A non-positive window must not make _maybe_rampup spin forever; rampup + # is simply disabled. + scope = _scope_manager( + settings={"BACKOFF_WINDOW": 0}, config={"id": "x", "rampup": True} + ) + scope.can_send(now=0.0) + scope.can_send(now=10_000.0) + assert scope._concurrency == scope._min_concurrency + def test_set_concurrency_fires_slot_event(self): scope = _scope_manager(config={"id": "x", "concurrency": 1}) scope.record_sent(now=0.0) From 883425005982091fab9bc76ead91a76b6cf98ce9 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 26 Jun 2026 22:53:11 +0200 Subject: [PATCH 19/55] WIP --- docs/topics/throttling.rst | 43 ++++++++--- scrapy/core/engine.py | 13 ++-- scrapy/settings/default_settings.py | 1 + .../templates/project/module/settings.py.tmpl | 7 ++ scrapy/throttling.py | 72 ++++++++++++++----- tests/test_engine.py | 12 ++++ tests/test_throttling.py | 41 +++++++++++ 7 files changed, 159 insertions(+), 30 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 8d3ae004e..565bbd09f 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -63,24 +63,30 @@ The main throttling :ref:`settings ` are: The wait time is measured from when the previous request was sent. -For example, with ``CONCURRENT_REQUESTS_PER_DOMAIN = 2``, ``DOWNLOAD_DELAY = 0.3``, -and ``DOWNLOAD_DELAY_PER_SLOT = 1.0``, sending 3 requests to the same domain -would result in: +For example, with ``DOWNLOAD_DELAY = 1.0`` (and, by default, a single download +slot per domain), requests to the same domain are sent at most once per second: .. code-block:: text - T=0.0s: Request 1 sent (slot 1) - T=0.3s: Request 2 sent (slot 2, respects same-domain delay) - T=0.6s: Request 3 must wait (same-domain delay satisfied, but slot 1 needs 1.0s) - T=1.0s: Request 3 sent (slot 1 can now be reused) + T=0.0s: Request 1 sent + T=1.0s: Request 2 sent + T=2.0s: Request 3 sent + +:setting:`DOWNLOAD_DELAY` (per :ref:`throttling scope `) and +:setting:`DOWNLOAD_DELAY_PER_SLOT` (per download slot) are enforced +independently. By default each domain is both its own scope and its own +download slot, so both apply to the same requests and the effective minimum +spacing is the larger of the two; they only differ when requests are grouped +into custom :ref:`scopes ` or download slots (via the +``download_slot`` request meta key). When configuring these settings, note that: - :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``. -- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``. - This happens because all slots must wait for the delay between requests, - preventing them from sending requests simultaneously. +- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, + because the next request to the domain is not sent until the delay elapses, + by which time the previous response has already arrived. .. [1] You can :ref:`customize ` how requests are grouped for throttling, but domain-based throttling works well in most cases. For @@ -1061,6 +1067,23 @@ Additional settings Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. +- .. setting:: THROTTLING_SCOPE_LIMIT + + :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) + + Maximum number of :ref:`throttling scope ` states kept + in memory at once, to bound memory usage on broad crawls that touch a large + number of scopes (e.g. domains). + + When the limit is exceeded, the least-recently-used idle scopes are evicted + (an evicted scope is recreated from its configuration the next time it is + needed). Scopes with in-flight requests or in active backoff are never + evicted, so the limit may be temporarily exceeded if that many scopes are + busy at once. Set to ``0`` to disable the limit. + + This complements :setting:`THROTTLING_SCOPE_MAX_IDLE`, which evicts scopes + by inactivity time rather than by count. + - .. setting:: THROTTLING_SCOPE_MAX_IDLE :setting:`THROTTLING_SCOPE_MAX_IDLE` (default: ``3600.0``) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7b761d80b..042468cd7 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -380,11 +380,16 @@ class ExecutionEngine: if delay_fn is None or not scheduler.has_pending_requests(): return delay = delay_fn() - if delay is None: + # ``delay`` is ``None`` when no pending request is time-blocked, and + # ``0`` when some request is ready right now but could not be sent (e.g. + # the downloader is at capacity). Neither case needs a timer: a freed + # slot already re-runs the loop from :meth:`_download`'s ``finally``, + # and arming a ``0``-second timer here would busy-loop the engine while + # the downloader stays full. Only a positive delay, i.e. a time-based + # gate that nothing else would wake us for, needs one. + if delay is None or delay <= 0: return - self._throttling_wakeup = call_later( - max(0.0, delay), self._slot.nextcall.schedule - ) + self._throttling_wakeup = call_later(delay, self._slot.nextcall.schedule) def needs_backout(self) -> bool: """Returns ``True`` if no more requests can be sent at the moment, or diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f19894658..586c2ecfd 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -598,6 +598,7 @@ THROTTLING_SCOPES = {} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLING_SCOPE_LIMIT = 100000 THROTTLING_SCOPE_MAX_IDLE = 3600.0 THROTTLING_DEBUG = False diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index e1ebdf707..64b27db68 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -9,5 +9,12 @@ NEWSPIDER_MODULE = "$project_name.spiders" # User-Agent header: #USER_AGENT = "$project_name (+https://your-domain.example)" +# Obey robots.txt rules: +ROBOTSTXT_OBEY = True + +# Throttle crawls to be polite to websites: +CONCURRENT_REQUESTS_PER_DOMAIN = 1 +DOWNLOAD_DELAY = 1 + # Set settings whose default value is deprecated to a future-proof value: FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index bdddd3d3b..ddb540f53 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -5,6 +5,7 @@ import datetime as dt import logging import random import time +from collections import OrderedDict from collections.abc import Awaitable, Callable, Iterable from email.utils import parsedate_to_datetime from functools import wraps @@ -45,7 +46,9 @@ def _parse_retry_after(response: Response) -> float | None: date = date.replace(tzinfo=dt.timezone.utc) now = dt.datetime.now(dt.timezone.utc) seconds_to_wait = (date - now).total_seconds() - return max(0, int(seconds_to_wait)) or None + # Keep sub-second precision (a date less than a second away must not be + # truncated to 0 and dropped); a past or present date yields no delay. + return max(0.0, seconds_to_wait) or None def _parse_ratelimit_reset(response: Response) -> float | None: @@ -485,7 +488,12 @@ class ThrottlingManager: self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict( "THROTTLING_SCOPES" ) - self._scope_managers: dict[ScopeID, ThrottlingScopeManagerProtocol] = {} + # Ordered by least-recently-used first (see _get_scope_manager), so the + # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT). + self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = ( + OrderedDict() + ) + self._scope_limit: int = crawler.settings.getint("THROTTLING_SCOPE_LIMIT") self._last_eviction: float | None = None # Concurrency slots reserved by acquire(), to be released once the # request finishes downloading. @@ -585,18 +593,46 @@ class ThrottlingManager: def _get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol: manager = self._scope_managers.get(scope_id) - if manager is None: - config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {})) - config.setdefault("id", scope_id) - manager_cls = ( - load_object(config["manager"]) - if "manager" in config - else self._default_scope_manager_cls - ) - manager = build_from_crawler(manager_cls, self.crawler, config) - self._scope_managers[scope_id] = manager + if manager is not None: + # Mark as most-recently-used for the LRU scope limit. + self._scope_managers.move_to_end(scope_id) + return manager + config: dict[str, Any] = dict(self._scopes_config.get(scope_id, {})) + config.setdefault("id", scope_id) + manager_cls = ( + load_object(config["manager"]) + if "manager" in config + else self._default_scope_manager_cls + ) + manager = cast( + "ThrottlingScopeManagerProtocol", + build_from_crawler(manager_cls, self.crawler, config), + ) + self._scope_managers[scope_id] = manager + self._enforce_scope_limit(scope_id) return manager + def _enforce_scope_limit(self, keep: ScopeID) -> None: + """Evict least-recently-used idle scopes while the number of live scope + managers exceeds :setting:`THROTTLING_SCOPE_LIMIT` (``0`` disables the + limit). + + LRU order is kept by :meth:`_get_scope_manager` moving each accessed + scope to the end, so the coldest scopes are at the front. Only scopes + that are idle (no in-flight requests and no active backoff) are evicted; + the just-created *keep* scope is never evicted. A scope evicted while + still throttling is simply recreated from its configuration the next + time it is needed. + """ + if self._scope_limit <= 0 or len(self._scope_managers) <= self._scope_limit: + return + now = time.monotonic() + for scope_id in list(self._scope_managers): + if len(self._scope_managers) <= self._scope_limit: + break + if scope_id != keep and self._scope_managers[scope_id].is_idle(now, 0): + del self._scope_managers[scope_id] + async def acquire(self, request: Request) -> None: # A throttling-aware scheduler reserves the request before handing it # to the engine, so there is nothing left to wait for or record here. @@ -1008,9 +1044,9 @@ class ThrottlingScopeManager: :ref:`backoff `, :ref:`rampup `, concurrency and :ref:`quotas `: - - A base :setting:`DOWNLOAD_DELAY`-style delay (``0`` by default, taken - from the scope ``"delay"`` config) is enforced between consecutive - requests for the scope. + - A base delay (the scope ``"delay"`` config, defaulting to + :setting:`DOWNLOAD_DELAY`) is enforced between consecutive requests for + the scope. - On a backoff trigger (a :setting:`BACKOFF_HTTP_CODES` response or a :setting:`BACKOFF_EXCEPTIONS` exception) the delay grows exponentially @@ -1043,7 +1079,11 @@ class ThrottlingScopeManager: settings = crawler.settings backoff: dict[str, Any] = config.get("backoff", {}) self._id: ScopeID = config.get("id", "") - self._base_delay: float = float(config.get("delay", 0.0)) + # The per-scope delay defaults to DOWNLOAD_DELAY; a scope can override + # it with its own "delay" config (see THROTTLING_SCOPES). + self._base_delay: float = float( + config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) + ) self._randomize: bool = bool( config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")) ) diff --git a/tests/test_engine.py b/tests/test_engine.py index 60d911009..032e5df2b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -694,6 +694,18 @@ class TestEngineThrottling: engine._maybe_arm_throttling_wakeup() assert engine._throttling_wakeup is None + def test_maybe_arm_throttling_wakeup_zero_delay(self, engine): + # A 0 delay means a request is ready but could not be sent (e.g. the + # downloader is at capacity); arming a 0-second timer would busy-loop + # the engine, so no timer must be armed. + scheduler = Mock() + scheduler.has_pending_requests.return_value = True + scheduler.next_request_delay.return_value = 0.0 + engine._slot = Mock() + engine._slot.scheduler = scheduler + engine._maybe_arm_throttling_wakeup() + assert engine._throttling_wakeup is None + def test_warn_delayed_requests(self, engine): engine._delayed_requests_warn_threshold = 1 engine._throttling_waiting = {Request("http://a.example")} diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 9ba6d51eb..d088d6c7e 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -415,6 +415,32 @@ class TestThrottlingManager: assert "idle.example" not in manager._scope_managers assert "active.example" in manager._scope_managers + def test_scope_limit_evicts_least_recently_used(self): + manager = _manager({"THROTTLING_SCOPE_LIMIT": 2}) + # Use three scopes in order; each send/done leaves them idle. + for scope_id in ("a.example", "b.example", "c.example"): + scope = manager._get_scope_manager(scope_id) + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + # The limit caps live managers at 2, dropping the least-recently-used. + assert set(manager._scope_managers) == {"b.example", "c.example"} + + def test_scope_limit_keeps_active_scopes(self): + manager = _manager({"THROTTLING_SCOPE_LIMIT": 1}) + # Two scopes with in-flight requests cannot be evicted, so the limit is + # exceeded rather than dropping a scope that still tracks a live send. + for scope_id in ("a.example", "b.example"): + manager._get_scope_manager(scope_id).record_sent(now=0.0) + assert set(manager._scope_managers) == {"a.example", "b.example"} + + def test_scope_limit_disabled(self): + manager = _manager({"THROTTLING_SCOPE_LIMIT": 0}) + for i in range(5): + scope = manager._get_scope_manager(f"{i}.example") + scope.record_sent(now=0.0) + scope.record_done(now=0.0) + assert len(manager._scope_managers) == 5 + class TestThrottlingScopeManager: def test_no_delay_by_default(self): @@ -431,6 +457,21 @@ class TestThrottlingScopeManager: assert scope.can_send(now=11.0) == pytest.approx(1.0) assert scope.can_send(now=12.0) == 0 + def test_base_delay_defaults_to_download_delay(self): + # With no explicit scope "delay", the base delay is DOWNLOAD_DELAY. + scope = _scope_manager( + {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, {"id": "x"} + ) + assert scope._base_delay == pytest.approx(2.0) + + def test_scope_delay_overrides_download_delay(self): + # An explicit scope "delay" overrides DOWNLOAD_DELAY. + scope = _scope_manager( + {"DOWNLOAD_DELAY": 2.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, + {"id": "x", "delay": 0.0}, + ) + assert scope._base_delay == pytest.approx(0.0) + def test_exponential_backoff(self): scope = _scope_manager( { From 184166d62958ae087cfed1ba489ab28982312bea Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 29 Jun 2026 10:58:18 +0200 Subject: [PATCH 20/55] Use get_* for getters --- scrapy/core/engine.py | 6 +++--- scrapy/core/scheduler.py | 6 +++--- scrapy/pqueues.py | 8 ++++---- scrapy/throttling.py | 20 ++++++++++---------- tests/test_engine.py | 10 +++++----- tests/test_pqueues.py | 20 ++++++++++---------- tests/test_scheduler.py | 4 ++-- tests/test_throttling.py | 24 ++++++++++++------------ 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 042468cd7..75a70ce9a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -140,7 +140,7 @@ class ExecutionEngine: self._scheduling: int = 0 # A coalesced wakeup timer, armed when a throttling-aware scheduler # reports that all pending requests are time-blocked (see - # ``next_request_delay``). + # ``get_next_request_delay``). self._throttling_wakeup: CallLaterResult | None = None self._delayed_requests_warn_threshold: int = self.settings.getint( "DELAYED_REQUESTS_WARN_THRESHOLD" @@ -376,7 +376,7 @@ class ExecutionEngine: """ assert self._slot is not None scheduler = self._slot.scheduler - delay_fn = getattr(scheduler, "next_request_delay", None) + delay_fn = getattr(scheduler, "get_next_request_delay", None) if delay_fn is None or not scheduler.has_pending_requests(): return delay = delay_fn() @@ -433,7 +433,7 @@ class ExecutionEngine: # scheduler instead of in _throttling_waiting, so it does not hit this # path; recommend it only when it is not already in use. scheduler = self._slot.scheduler if self._slot is not None else None - if scheduler is None or not hasattr(scheduler, "next_request_delay"): + if scheduler is None or not hasattr(scheduler, "get_next_request_delay"): recommendation = ( " Consider switching to scrapy.core.scheduler.ThrottlingAwareScheduler." ) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 76eb3b5fe..90a892dc3 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -544,7 +544,7 @@ class ThrottlingAwareScheduler(Scheduler): def open(self, spider: Spider) -> Deferred[None] | None: result = super().open(spider) - if not hasattr(self.mqs, "next_request_delay"): + if not hasattr(self.mqs, "get_next_request_delay"): raise ValueError( f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be " f"set to a throttling-aware priority queue such as " @@ -619,7 +619,7 @@ class ThrottlingAwareScheduler(Scheduler): return False return True - def next_request_delay(self) -> float | None: + def get_next_request_delay(self) -> float | None: """Return the minimum number of seconds until some pending request becomes sendable because a time-based throttling gate opens, or ``None`` if no pending request is time-blocked. @@ -630,6 +630,6 @@ class ThrottlingAwareScheduler(Scheduler): delays = [ delay for pq in (self.mqs, self.dqs) - if pq is not None and (delay := pq.next_request_delay()) is not None # type: ignore[attr-defined] + if pq is not None and (delay := pq.get_next_request_delay()) is not None # type: ignore[attr-defined] ] return min(delays) if delays else None diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 5843c9e9e..dfee687b4 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -603,7 +603,7 @@ class ThrottlingAwarePriorityQueue: Among the sendable queues (those whose scope set can be sent right now), the one whose head has the highest request priority is chosen; ties are broken by ascending load (the maximum - :meth:`~scrapy.throttling.ThrottlingManagerProtocol.scope_load` over the + :meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scope_load` over the scopes of the queue), i.e. by preferring the least-busy scopes. """ self._promote_ready(time.monotonic()) @@ -614,7 +614,7 @@ class ThrottlingAwarePriorityQueue: if head is None or not self._throttler.is_ready(head): continue load = max( - (self._throttler.scope_load(scope_id) for scope_id in scope_set), + (self._throttler.get_scope_load(scope_id) for scope_id in scope_set), default=0.0, ) sort_key = (queue.priority(head), load) @@ -641,7 +641,7 @@ class ThrottlingAwarePriorityQueue: return None return selected[1].peek() - def next_request_delay(self) -> float | None: + def get_next_request_delay(self) -> float | None: now = time.monotonic() self._promote_ready(now) delay: float | None = None @@ -651,7 +651,7 @@ class ThrottlingAwarePriorityQueue: continue if self._throttler.is_ready(head): return 0.0 - head_delay = self._throttler.time_until_ready(head) + head_delay = self._throttler.get_time_until_ready(head) if head_delay is None: continue if delay is None or head_delay < delay: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index ddb540f53..ebb1a903e 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -342,7 +342,7 @@ class ThrottlingManagerProtocol(Protocol): returned ``True``). The reservation is released by :meth:`release`. """ - def time_until_ready(self, request: Request) -> float | None: + def get_time_until_ready(self, request: Request) -> float | None: """Return the number of seconds until every time-based gate of *request* would be open, or ``None`` if no time-based gate is currently blocking it (only a concurrency slot could be). @@ -352,7 +352,7 @@ class ThrottlingManagerProtocol(Protocol): requests are time-blocked. """ - def scope_load(self, scope_id: str) -> float: + def get_scope_load(self, scope_id: str) -> float: """Return the current load of the scope identified by *scope_id*: its active sends divided by its concurrency limit (or by the global :setting:`CONCURRENT_REQUESTS` when the scope has no explicit limit). @@ -510,7 +510,7 @@ class ThrottlingManager: It backs :meth:`get_scopes` and is also the fallback for the synchronous readiness methods (:meth:`is_ready`, :meth:`reserve`, - :meth:`time_until_ready`) when no scopes were persisted on + :meth:`get_time_until_ready`) when no scopes were persisted on ``request.meta`` by an earlier :meth:`get_scopes` call (which normally happens at enqueue time and survives disk restores; see :func:`scope_cache`). Subclasses whose :meth:`get_scopes` cannot be @@ -712,7 +712,7 @@ class ThrottlingManager: manager.record_sent(amount=value) self._reserved[request] = managers - def time_until_ready(self, request: Request) -> float | None: + def get_time_until_ready(self, request: Request) -> float | None: now = time.monotonic() wait = max(0.0, self._request_delay_deadline(request, now) - now) for scope_id, value in self._cached_scope_values(request): @@ -720,8 +720,8 @@ class ThrottlingManager: wait = max(wait, manager.can_send(now=now, amount=value)) return wait if wait > 0 else None - def scope_load(self, scope_id: ScopeID) -> float: - return self._get_scope_manager(scope_id).load() + def get_scope_load(self, scope_id: ScopeID) -> float: + return self._get_scope_manager(scope_id).get_load() def get_request_delay(self, request: Request, now: float | None = None) -> float: now = time.monotonic() if now is None else now @@ -760,7 +760,7 @@ class ThrottlingManager: This is the readiness-API counterpart of :meth:`_delay_request`: a throttling-aware scheduler gates requests through :meth:`is_ready` and - :meth:`time_until_ready` instead of awaiting :meth:`acquire`, so the + :meth:`get_time_until_ready` instead of awaiting :meth:`acquire`, so the delay is enforced by holding back the request until this deadline rather than by sleeping. The deadline is computed once, the first time the request reaches the gate, and stored so later polls reuse it. @@ -996,7 +996,7 @@ class ThrottlingScopeManagerProtocol(Protocol): Return ``False`` when no concurrency limit is enforced. """ - def load(self) -> float: + def get_load(self) -> float: """Return the current load of this scope: a non-negative number, with ``1.0`` meaning "as busy as its concurrency limit allows". @@ -1123,7 +1123,7 @@ class ThrottlingScopeManager: else: self._concurrency = None # Used as the load denominator when the scope enforces no explicit - # concurrency limit (see load()). + # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") # Quota. @@ -1281,7 +1281,7 @@ class ThrottlingScopeManager: def concurrency_blocked(self) -> bool: return self._concurrency is not None and self._active >= self._concurrency - def load(self) -> float: + def get_load(self) -> float: limit = ( self._concurrency if self._concurrency is not None diff --git a/tests/test_engine.py b/tests/test_engine.py index 032e5df2b..2b15c40d1 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -677,7 +677,7 @@ class TestEngineThrottling: def test_maybe_arm_throttling_wakeup_arms_timer(self, engine): scheduler = Mock() scheduler.has_pending_requests.return_value = True - scheduler.next_request_delay.return_value = 5.0 + scheduler.get_next_request_delay.return_value = 5.0 engine._slot = Mock() engine._slot.scheduler = scheduler engine._maybe_arm_throttling_wakeup() @@ -688,7 +688,7 @@ class TestEngineThrottling: def test_maybe_arm_throttling_wakeup_no_delay(self, engine): scheduler = Mock() scheduler.has_pending_requests.return_value = True - scheduler.next_request_delay.return_value = None + scheduler.get_next_request_delay.return_value = None engine._slot = Mock() engine._slot.scheduler = scheduler engine._maybe_arm_throttling_wakeup() @@ -700,7 +700,7 @@ class TestEngineThrottling: # the engine, so no timer must be armed. scheduler = Mock() scheduler.has_pending_requests.return_value = True - scheduler.next_request_delay.return_value = 0.0 + scheduler.get_next_request_delay.return_value = 0.0 engine._slot = Mock() engine._slot.scheduler = scheduler engine._maybe_arm_throttling_wakeup() @@ -710,7 +710,7 @@ class TestEngineThrottling: engine._delayed_requests_warn_threshold = 1 engine._throttling_waiting = {Request("http://a.example")} engine._slot = Mock() - # A scheduler without next_request_delay is not throttling-aware, so the + # A scheduler without get_next_request_delay is not throttling-aware, so the # warning recommends switching to one. engine._slot.scheduler = Mock(spec=BaseScheduler) with LogCapture() as log: @@ -726,7 +726,7 @@ class TestEngineThrottling: engine._delayed_requests_warn_threshold = 1 engine._throttling_waiting = {Request("http://a.example")} engine._slot = Mock() - # A throttling-aware scheduler (one with next_request_delay) holds + # A throttling-aware scheduler (one with get_next_request_delay) holds # throttled requests itself, so no switch is recommended. engine._slot.scheduler = Mock() with LogCapture() as log: diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index 5fc4207e4..c45687a3f 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -364,7 +364,7 @@ class TestThrottlingAwarePriorityQueue: # The blocked second slow request stays in the queue. assert None in urls assert len(queue) == 1 - delay = queue.next_request_delay() + delay = queue.get_next_request_delay() assert delay is not None assert delay == pytest.approx(1000.0, abs=1.0) @@ -385,7 +385,7 @@ class TestThrottlingAwarePriorityQueue: assert "http://fast.com/1" in urls assert None in urls assert len(queue) == 1 - delay = queue.next_request_delay() + delay = queue.get_next_request_delay() assert delay is not None assert delay == pytest.approx(1000.0, abs=1.0) @@ -407,7 +407,7 @@ class TestThrottlingAwarePriorityQueue: # The delayed request is not lost, just not poppable yet. assert queue.pop() is None assert len(queue) == 1 - assert queue.next_request_delay() == pytest.approx(1000.0, abs=1.0) + assert queue.get_next_request_delay() == pytest.approx(1000.0, abs=1.0) @coroutine_test async def test_delayed_request_promoted_when_due(self): @@ -508,7 +508,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler(Spider) queue = self._queue(crawler) assert queue.pop() is None - assert queue.next_request_delay() is None + assert queue.get_next_request_delay() is None await self._push(queue, crawler, Request("http://a.com/1")) assert queue.close() != {} @@ -525,23 +525,23 @@ class TestThrottlingAwarePriorityQueue: assert len(queue) == 1 @coroutine_test - async def test_next_request_delay_zero_when_ready(self): + async def test_get_next_request_delay_zero_when_ready(self): crawler = get_crawler(Spider) queue = self._queue(crawler) await self._push(queue, crawler, Request("http://a.com/1")) # A sendable head means no wait is needed. - assert queue.next_request_delay() == 0.0 + assert queue.get_next_request_delay() == 0.0 @coroutine_test - async def test_next_request_delay_ignores_empty_queues(self): + async def test_get_next_request_delay_ignores_empty_queues(self): crawler = get_crawler(Spider) queue = self._queue(crawler) # An empty (but still registered) internal queue is skipped. queue.pqueues[frozenset({"a.com"})] = queue.pqfactory(frozenset({"a.com"})) - assert queue.next_request_delay() is None + assert queue.get_next_request_delay() is None @coroutine_test - async def test_next_request_delay_keeps_minimum(self): + async def test_get_next_request_delay_keeps_minimum(self): crawler = get_crawler( Spider, settings_dict={ @@ -563,7 +563,7 @@ class TestThrottlingAwarePriorityQueue: queue.pop() # Both scopes are now time-blocked; the smaller per-scope delay wins, # so the larger one exercises the "not below the running minimum" branch. - delay = queue.next_request_delay() + delay = queue.get_next_request_delay() assert delay == pytest.approx(10.0, abs=1.0) @coroutine_test diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 12dc61461..d1d56bbcb 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -498,7 +498,7 @@ class TestThrottlingAwareScheduler: assert first is not None assert scheduler.next_request() is None assert scheduler.has_pending_requests() - assert scheduler.next_request_delay() == pytest.approx(1000.0, abs=1.0) + assert scheduler.get_next_request_delay() == pytest.approx(1000.0, abs=1.0) scheduler.close("finished") @coroutine_test @@ -512,7 +512,7 @@ class TestThrottlingAwareScheduler: assert scheduler.next_request() is not None assert scheduler.next_request() is None # A purely concurrency-blocked state has no time-based wakeup. - assert scheduler.next_request_delay() is None + assert scheduler.get_next_request_delay() is None scheduler.close("finished") @coroutine_test diff --git a/tests/test_throttling.py b/tests/test_throttling.py index d088d6c7e..abcb29604 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -762,7 +762,7 @@ class TestThrottlingManagerReadiness: manager.reserve(first) # The base delay now blocks any further request for the scope. assert manager.is_ready(second) is False - assert manager.time_until_ready(second) == pytest.approx(100.0, abs=1.0) + assert manager.get_time_until_ready(second) == pytest.approx(100.0, abs=1.0) @coroutine_test async def test_throttling_delay_blocks_until_deadline(self): @@ -772,7 +772,7 @@ class TestThrottlingManagerReadiness: # The per-request delay holds back the request even though its scope is # otherwise unconstrained. assert manager.is_ready(request) is False - assert manager.time_until_ready(request) == pytest.approx(100.0, abs=1.0) + assert manager.get_time_until_ready(request) == pytest.approx(100.0, abs=1.0) # The deadline is computed once and reused by later polls. deadline = request.meta["_throttling_delay_deadline"] assert manager.is_ready(request) is False @@ -813,7 +813,7 @@ class TestThrottlingManagerReadiness: # response header does. manager.delay_scope("example.com", 50.0) assert manager.is_ready(request) is False - assert manager.time_until_ready(request) == pytest.approx(50.0, abs=1.0) + assert manager.get_time_until_ready(request) == pytest.approx(50.0, abs=1.0) @coroutine_test async def test_delay_scope_bypasses_max_delay(self): @@ -825,7 +825,7 @@ class TestThrottlingManagerReadiness: request = Request("http://example.com/a") await manager.get_scopes(request) manager.delay_scope("example.com", 1000.0) - assert manager.time_until_ready(request) == pytest.approx(1000.0, abs=1.0) + assert manager.get_time_until_ready(request) == pytest.approx(1000.0, abs=1.0) @coroutine_test async def test_reserve_blocks_on_concurrency(self): @@ -837,7 +837,7 @@ class TestThrottlingManagerReadiness: manager.reserve(first) assert manager.is_ready(second) is False # Pure concurrency blocking is not time-gated. - assert manager.time_until_ready(second) is None + assert manager.get_time_until_ready(second) is None manager.release(first) assert manager.is_ready(second) is True @@ -859,20 +859,20 @@ class TestThrottlingManagerReadiness: assert scope._active == 1 # reserve recorded exactly one send @coroutine_test - async def test_scope_load(self): + async def test_get_scope_load(self): manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 4}}}) - assert manager.scope_load("example.com") == 0.0 + assert manager.get_scope_load("example.com") == 0.0 request = Request("http://example.com/1") await manager.get_scopes(request) manager.reserve(request) - assert manager.scope_load("example.com") == pytest.approx(0.25) + assert manager.get_scope_load("example.com") == pytest.approx(0.25) - def test_scope_load_falls_back_to_global_concurrency(self): + def test_get_scope_load_falls_back_to_global_concurrency(self): manager = _manager({"CONCURRENT_REQUESTS": 8}) # A scope with no explicit concurrency limit uses CONCURRENT_REQUESTS. request = Request("http://example.com/1") manager.reserve(request) - assert manager.scope_load("example.com") == pytest.approx(1 / 8) + assert manager.get_scope_load("example.com") == pytest.approx(1 / 8) class TestParseRateHeaders: @@ -971,11 +971,11 @@ class TestThrottlingManagerEdges: await manager.acquire(request) assert request not in manager._reserved - def test_scope_load_without_concurrency_limit(self): + def test_get_scope_load_without_concurrency_limit(self): manager = _manager({"CONCURRENT_REQUESTS": 0}) # CONCURRENT_REQUESTS is 0, so the load denominator is 0 and the load is # reported as 0 instead of raising. - assert manager.scope_load("example.com") == 0.0 + assert manager.get_scope_load("example.com") == 0.0 @coroutine_test async def test_acquire_logs_and_waits_for_delay(self): From 8dfb548415eedac8a0b1a0f64e39788a27d97e27 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 29 Jun 2026 16:29:31 +0200 Subject: [PATCH 21/55] WIP --- docs/topics/throttling.rst | 37 +- scrapy/core/downloader/__init__.py | 340 +++++++++--------- .../downloader/handlers/_base_streaming.py | 12 +- scrapy/core/downloader/handlers/http11.py | 12 +- scrapy/crawler.py | 42 +++ scrapy/pqueues.py | 28 +- scrapy/settings/__init__.py | 16 + scrapy/settings/default_settings.py | 2 +- scrapy/throttling.py | 27 +- scrapy/utils/test.py | 4 +- tests/test_core_downloader.py | 37 +- tests/test_crawler.py | 55 +++ tests/test_downloaderslotssettings.py | 135 +++---- tests/test_engine.py | 2 +- tests/test_pqueues.py | 48 +-- tests/test_throttling.py | 6 +- 16 files changed, 469 insertions(+), 334 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 565bbd09f..07b0fa0a5 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -38,48 +38,15 @@ The main throttling :ref:`settings ` are: - .. setting:: DOWNLOAD_DELAY - :setting:`DOWNLOAD_DELAY` (default: ``1`` (:ref:`fallback `: ``0``)) + :setting:`DOWNLOAD_DELAY` (default: ``1`` + (:ref:`fallback `: ``0``)) Minimum seconds between any two requests to the same domain. - Even if you have multiple slots, requests to the same domain cannot be sent - more frequently than this delay. - To target a specific number of requests per minute (RPM) *per domain*, set this to ``60 / RPM``. For example, ``DOWNLOAD_DELAY = 1.0`` for 60 RPM, or ``DOWNLOAD_DELAY = 2.0`` for 30 RPM. -- .. setting:: DOWNLOAD_DELAY_PER_SLOT - - :setting:`DOWNLOAD_DELAY_PER_SLOT` (default: ``None``) - - Minimum seconds to wait between two consecutive requests sent to the same - download slot. Unlike :setting:`DOWNLOAD_DELAY`, which applies per domain - (:ref:`throttling scope `), this delay is per slot. - - When ``None`` (default), the per-slot delay falls back to - :setting:`DOWNLOAD_DELAY`, preserving the historical behavior where - :setting:`DOWNLOAD_DELAY` was enforced per slot. - - The wait time is measured from when the previous request was sent. - -For example, with ``DOWNLOAD_DELAY = 1.0`` (and, by default, a single download -slot per domain), requests to the same domain are sent at most once per second: - -.. code-block:: text - - T=0.0s: Request 1 sent - T=1.0s: Request 2 sent - T=2.0s: Request 3 sent - -:setting:`DOWNLOAD_DELAY` (per :ref:`throttling scope `) and -:setting:`DOWNLOAD_DELAY_PER_SLOT` (per download slot) are enforced -independently. By default each domain is both its own scope and its own -download slot, so both apply to the same requests and the effective minimum -spacing is the larger of the two; they only differ when requests are grouped -into custom :ref:`scopes ` or download slots (via the -``download_slot`` request meta key). - When configuring these settings, note that: - :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``. diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index a9778d1d7..e22b1192d 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,127 +1,208 @@ from __future__ import annotations import random -from collections import deque +import warnings +from collections.abc import Iterator, Mapping from dataclasses import dataclass, field -from datetime import datetime -from time import monotonic from typing import TYPE_CHECKING, Any -from twisted.internet.defer import Deferred, inlineCallbacks -from twisted.python.failure import Failure +from twisted.internet.defer import inlineCallbacks from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.resolver import dnscache -from scrapy.utils.asyncio import ( - AsyncioLoopingCall, - CallLaterResult, - call_later, - create_looping_call, -) from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.defer import ( - _defer_sleep_async, - _schedule_coro, - deferred_from_coro, - maybe_deferred_to_future, -) -from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute +from scrapy.utils.defer import _defer_sleep_async, deferred_from_coro +from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from collections.abc import Generator - from twisted.internet.task import LoopingCall + from twisted.internet.defer import Deferred from scrapy.crawler import Crawler from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager + from scrapy.throttling import ThrottlingScopeManagerProtocol @dataclass(slots=True, eq=False) -class Slot: +class _Slot: """Downloader slot""" - concurrency: int - delay: float - randomize_delay: bool - active: set[Request] = field(default_factory=set, init=False, repr=False) - queue: deque[tuple[Request, Deferred[Response]]] = field( - default_factory=deque, init=False, repr=False - ) transferring: set[Request] = field(default_factory=set, init=False, repr=False) lastseen: float = field(default=0, init=False, repr=False) - latercall: CallLaterResult | None = field(default=None, init=False, repr=False) + + +Slot = create_deprecated_class( + "Slot", + _Slot, + old_class_path="scrapy.core.downloader.Slot", + subclass_warn_message=("{cls} inherits from the deprecated Slot class."), + instance_warn_message=("The Slot class is deprecated."), +) + + +class _DeprecatedSlotView: + """Deprecated per-domain slot view backed by the downloader and throttler.""" + + __slots__ = ("_downloader", "_key", "_scope") + + def __init__( + self, + downloader: Downloader, + key: str, + scope: ThrottlingScopeManagerProtocol | None, + ) -> None: + self._downloader = downloader + self._key = key + self._scope = scope + + @property + def active(self) -> set[Request]: + return { + r + for r in self._downloader.active + if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key + } + + @property + def transferring(self) -> set[Request]: + return { + r + for r in self._downloader._transferring + if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key + } + + @property + def lastseen(self) -> float: + return 0.0 + + @property + def delay(self) -> float: + if self._scope is not None: + return self._scope._delay # type: ignore[union-attr] + return 0.0 + + @delay.setter + def delay(self, value: float) -> None: + if self._scope is not None: + self._scope._delay = value # type: ignore[union-attr] + + @property + def randomize_delay(self) -> bool: + if self._scope is not None: + return self._scope._randomize # type: ignore[union-attr] + return False + + @property + def concurrency(self) -> int: + warnings.warn( + "Slot.concurrency is deprecated. Per-slot concurrency limits are " + "now managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + if self._scope is not None: + return self._scope._concurrency or 0 # type: ignore[union-attr] + return 0 def free_transfer_slots(self) -> int: - return self.concurrency - len(self.transferring) + concurrency = self._scope._concurrency if self._scope is not None else 0 # type: ignore[union-attr] + return concurrency - len(self.transferring) def download_delay(self) -> float: + delay = self.delay if self.randomize_delay: - return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311 - return self.delay + return random.uniform(0.5 * delay, 1.5 * delay) # noqa: S311 + return delay def close(self) -> None: - if self.latercall: - self.latercall.cancel() - self.latercall = None + pass + + def __repr__(self) -> str: + return f"_DeprecatedSlotView({self._key!r})" def __str__(self) -> str: - return ( - f"" + return f"_DeprecatedSlotView({self._key!r})" + + +class _DeprecatedSlotsView(Mapping): + """Deprecated mapping view of active downloads, keyed by slot name.""" + + __slots__ = ("_downloader", "_throttler") + + def __init__(self, downloader: Downloader, throttler: Any) -> None: + self._downloader = downloader + self._throttler = throttler + + def _active_keys(self) -> set[str]: + return { + r.meta[Downloader.DOWNLOAD_SLOT] + for r in self._downloader.active + if Downloader.DOWNLOAD_SLOT in r.meta + } + + def __getitem__(self, key: str) -> _DeprecatedSlotView: + if key not in self._active_keys(): + raise KeyError(key) + scope = ( + self._throttler._get_scope_manager(key) + if self._throttler is not None + else None ) + return _DeprecatedSlotView(self._downloader, key, scope) + def __iter__(self) -> Iterator[str]: + return iter(self._active_keys()) -def _get_concurrency_delay( - concurrency: int, spider: Spider, settings: BaseSettings -) -> tuple[int, float]: - delay: float = settings.getfloat("DOWNLOAD_DELAY") - if hasattr(spider, "download_delay"): - delay = spider.download_delay - if settings.get("DOWNLOAD_DELAY_PER_SLOT") is not None: - delay = settings.getfloat("DOWNLOAD_DELAY_PER_SLOT") + def __len__(self) -> int: + return len(self._active_keys()) - if hasattr(spider, "max_concurrent_requests"): # pragma: no cover - warn_on_deprecated_spider_attribute( - "max_concurrent_requests", "CONCURRENT_REQUESTS" - ) - concurrency = spider.max_concurrent_requests - - return concurrency, delay + def __contains__(self, key: object) -> bool: + return key in self._active_keys() class Downloader: DOWNLOAD_SLOT = "download_slot" - _SLOT_GC_INTERVAL: float = 60.0 # seconds def __init__(self, crawler: Crawler): self.crawler: Crawler = crawler self.settings: BaseSettings = crawler.settings self.signals: SignalManager = crawler.signals - self.slots: dict[str, Slot] = {} self.active: set[Request] = set() + self._transferring: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") - self.domain_concurrency: int = self.settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" - ) self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") - self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) - self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( "DOWNLOAD_SLOTS" ) + if self.per_slot_settings: + warnings.warn( + "The DOWNLOAD_SLOTS setting is deprecated. Use THROTTLING_SCOPES for " + "per-domain configuration instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + for slot_settings in self.per_slot_settings.values(): + for deprecated_key in ("concurrency", "delay", "randomize_delay"): + if deprecated_key in slot_settings: + warnings.warn( + f"The '{deprecated_key}' key in DOWNLOAD_SLOTS is deprecated." + " Use THROTTLING_SCOPES to configure per-domain settings" + " instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) @inlineCallbacks @_warn_spider_arg @@ -142,94 +223,53 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency + @property + def slots(self) -> _DeprecatedSlotsView: + warnings.warn( + "Downloader.slots is deprecated. Use the throttling manager API instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return _DeprecatedSlotsView(self, self.crawler.throttler) + @_warn_spider_arg def _get_slot( self, request: Request, spider: Spider | None = None - ) -> tuple[str, Slot]: - key = self.get_slot_key(request) - if key not in self.slots: - assert self.crawler.spider - slot_settings = self.per_slot_settings.get(key, {}) - conc = self.ip_concurrency or self.domain_concurrency - conc, delay = _get_concurrency_delay( - conc, self.crawler.spider, self.settings - ) - conc, delay = ( - slot_settings.get("concurrency", conc), - slot_settings.get("delay", delay), - ) - randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay) - new_slot = Slot(conc, delay, randomize_delay) - self.slots[key] = new_slot - self._start_slot_gc() + ) -> tuple[str, _DeprecatedSlotView]: + key = self._get_slot_key(request) + scope = ( + self.crawler.throttler._get_scope_manager(key) + if self.crawler.throttler is not None + else None + ) + return key, _DeprecatedSlotView(self, key, scope) - return key, self.slots[key] + def _get_slot_key(self, request: Request) -> str: + throttler = self.crawler.throttler + if throttler is not None: + return throttler.get_slot_key(request) + return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) - if meta_slot is not None: - return meta_slot - - key = urlparse_cached(request).hostname or "" + key = urlparse_cached(request).netloc or "" if self.ip_concurrency: key = dnscache.get(key, key) - return key - # passed as download_func into self.middleware.download() in self.fetch() async def _enqueue_request(self, request: Request) -> Response: - key, slot = self._get_slot(request) + key = self._get_slot_key(request) request.meta[self.DOWNLOAD_SLOT] = key - slot.active.add(request) self.signals.send_catch_log( signal=signals.request_reached_downloader, request=request, spider=self.crawler.spider, ) - d: Deferred[Response] = Deferred() - slot.queue.append((request, d)) - self._process_queue(slot) + return await self._download(request) + + async def _download(self, request: Request) -> Response: + self._transferring.add(request) try: - return await maybe_deferred_to_future(d) # fired in _wait_for_download() - finally: - slot.active.remove(request) - - def _process_queue(self, slot: Slot) -> None: - if slot.latercall: - # block processing until slot.latercall is called - return - - # Delay queue processing if a download_delay is configured - now = monotonic() - delay = slot.download_delay() - if delay: - penalty = delay - now + slot.lastseen - if penalty > 0: - slot.latercall = call_later(penalty, self._latercall, slot) - return - - # Process enqueued requests if there are free slots to transfer for this slot - while slot.queue and slot.free_transfer_slots() > 0: - slot.lastseen = now - request, queue_dfd = slot.queue.popleft() - _schedule_coro(self._wait_for_download(slot, request, queue_dfd)) - # prevent burst if inter-request delays were configured - if delay: - self._process_queue(slot) - break - - def _latercall(self, slot: Slot) -> None: - slot.latercall = None - self._process_queue(slot) - - async def _download(self, slot: Slot, request: Request) -> Response: - # The order is very important for the following logic. Do not change! - slot.transferring.add(request) - try: - # 1. Download the response response: Response = await self.handlers.download_request_async(request) - # 2. Notify response_downloaded listeners about the recent download - # before querying queue for next request self.signals.send_catch_log( signal=signals.response_downloaded, response=response, @@ -241,46 +281,12 @@ class Downloader: await _defer_sleep_async() raise finally: - # 3. After response arrives, remove the request from transferring - # state to free up the transferring slot so it can be used by the - # following requests (perhaps those which came from the downloader - # middleware itself) - slot.transferring.remove(request) - self._process_queue(slot) + self._transferring.discard(request) self.signals.send_catch_log( signal=signals.request_left_downloader, request=request, spider=self.crawler.spider, ) - async def _wait_for_download( - self, slot: Slot, request: Request, queue_dfd: Deferred[Response] - ) -> None: - try: - response = await self._download(slot, request) - except Exception: - queue_dfd.errback(Failure()) - else: - queue_dfd.callback(response) # awaited in _enqueue_request() - def close(self) -> None: - self._stop_slot_gc() - for slot in self.slots.values(): - slot.close() - - def _slot_gc(self, age: float = 60) -> None: - mintime = monotonic() - age - for key, slot in list(self.slots.items()): - if not slot.active and slot.lastseen + slot.delay < mintime: - self.slots.pop(key).close() - - def _start_slot_gc(self) -> None: - if self._slot_gc_loop: - return - self._slot_gc_loop = create_looping_call(self._slot_gc) - self._slot_gc_loop.start(self._SLOT_GC_INTERVAL, now=False) - - def _stop_slot_gc(self) -> None: - if self._slot_gc_loop: - self._slot_gc_loop.stop() - self._slot_gc_loop = None + pass diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py index 5a669c732..b7aba8fd0 100644 --- a/scrapy/core/downloader/handlers/_base_streaming.py +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -86,8 +86,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") # these are useful for many handlers but used in different ways by them self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") - self._pool_size_per_host: int = crawler.settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" + scope_concurrencies = [ + scope["concurrency"] + for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() + if "concurrency" in scope + ] + self._pool_size_per_host: int = max( + [ + crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), + *scope_concurrencies, + ] ) @staticmethod diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1fc504c59..c399551e7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,8 +92,16 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): from twisted.internet import reactor self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) - self._pool.maxPersistentPerHost = crawler.settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" + scope_concurrencies = [ + scope["concurrency"] + for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() + if "concurrency" in scope + ] + self._pool.maxPersistentPerHost = max( + [ + crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), + *scope_concurrencies, + ] ) self._pool._factory.noisy = False diff --git a/scrapy/crawler.py b/scrapy/crawler.py index c18e061f8..492877e7a 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -98,6 +98,8 @@ class Crawler: return self.addons.load_settings(self.settings) + self._warn_on_deprecated_default_settings() + self._apply_spider_download_delay() self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -157,6 +159,46 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) + def _apply_spider_download_delay(self) -> None: + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, "download_delay"): + return + delay_prio = self.settings.getpriority("DOWNLOAD_DELAY") or 0 + if delay_prio >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + "The 'download_delay' spider attribute is deprecated. " + "It is also being ignored because DOWNLOAD_DELAY is already set " + "at spider or higher priority. Remove the 'download_delay' " + "attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + else: + warnings.warn( + "The 'download_delay' spider attribute is deprecated. Use the " + "DOWNLOAD_DELAY setting or per-domain THROTTLING_SCOPES instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.settings.set( + "DOWNLOAD_DELAY", spider.download_delay, priority="spider" + ) + + def _warn_on_deprecated_default_settings(self) -> None: + default_priority = SETTINGS_PRIORITIES["default"] + for setting_name, current_default, future_default in ( + ("THROTTLING_SCOPE_CONCURRENCY", 8, 1), + ): + if self.settings.getpriority(setting_name) == default_priority: + warnings.warn( + f"The default value of {setting_name} will change from " + f"{current_default!r} to {future_default!r} in a future " + f"Scrapy version. Explicitly set {setting_name} in your " + f"settings to silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index dfee687b4..4394f7407 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -268,18 +268,20 @@ class DownloaderInterface: def __init__(self, crawler: Crawler): assert crawler.engine self.downloader: Downloader = crawler.engine.downloader + self._throttler: ThrottlingManagerProtocol | None = crawler.throttler - def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]: - return [(self._active_downloads(slot), slot) for slot in possible_slots] + def stats(self, possible_slots: Iterable[str]) -> list[tuple[float, str]]: + return [(self._slot_load(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: + if self._throttler is not None: + return self._throttler.get_slot_key(request) return self.downloader.get_slot_key(request) - def _active_downloads(self, slot: str) -> int: - """Return a number of requests in a Downloader for a given slot""" - if slot not in self.downloader.slots: - return 0 - return len(self.downloader.slots[slot].active) + def _slot_load(self, slot: str) -> float: + if self._throttler is not None: + return self._throttler.get_scope_load(slot) + return 0.0 class DownloaderAwarePriorityQueue: @@ -363,19 +365,19 @@ class DownloaderAwarePriorityQueue: for slot, startprios in slot_startprios.items(): self.pqueues[slot] = self.pqfactory(slot, startprios) - def _next_slot(self, stats: list[tuple[int, str]], *, update_state: bool) -> str: + def _next_slot(self, stats: list[tuple[float, str]], *, update_state: bool) -> str: last = self._last_selected_slot - min_active: int | None = None + min_load: float | None = None best_slot: str | None = None best_slot_after_last: str | None = None - for active, slot in stats: - if min_active is None or active < min_active: - min_active = active + for load, slot in stats: + if min_load is None or load < min_load: + min_load = load best_slot = slot best_slot_after_last = None if last is not None and slot > last: best_slot_after_last = slot - elif active == min_active: + elif load == min_load: if best_slot is None or slot < best_slot: best_slot = slot if ( diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index be298e9b1..833c71f44 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -166,6 +166,22 @@ class BaseSettings(MutableMapping[str, Any]): stacklevel=2, ) + if name == "THROTTLING_SCOPE_CONCURRENCY": + per_domain_prio = self.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") or 0 + new_prio = self.getpriority(name) or 0 + if ( + per_domain_prio > SETTINGS_PRIORITIES["default"] + and new_prio <= SETTINGS_PRIORITIES["default"] + ): + warnings.warn( + "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " + "THROTTLING_SCOPE_CONCURRENCY instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + per_domain_val = self["CONCURRENT_REQUESTS_PER_DOMAIN"] + return per_domain_val if per_domain_val is not None else default + return self[name] if self[name] is not None else default def getbool(self, name: str, default: bool = False) -> bool: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 586c2ecfd..6bfd73777 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -294,7 +294,6 @@ DNS_TIMEOUT = 60 DOWNLOAD_BIND_ADDRESS = None DOWNLOAD_DELAY = 0 -DOWNLOAD_DELAY_PER_SLOT = None DOWNLOAD_FAIL_ON_DATALOSS = True @@ -598,6 +597,7 @@ THROTTLING_SCOPES = {} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLING_SCOPE_CONCURRENCY = 8 THROTTLING_SCOPE_LIMIT = 100000 THROTTLING_SCOPE_MAX_IDLE = 3600.0 THROTTLING_DEBUG = False diff --git a/scrapy/throttling.py b/scrapy/throttling.py index ebb1a903e..040e15a28 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -5,6 +5,7 @@ import datetime as dt import logging import random import time +import warnings from collections import OrderedDict from collections.abc import Awaitable, Callable, Iterable from email.utils import parsedate_to_datetime @@ -16,6 +17,7 @@ from twisted.internet.defer import Deferred from typing_extensions import NotRequired, Self from scrapy import signals +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object @@ -352,6 +354,15 @@ class ThrottlingManagerProtocol(Protocol): requests are time-blocked. """ + def get_slot_key(self, request: Request) -> str: + """Return a single string key for *request*, derived from its scopes. + + For a single scope this is the scope ID itself; for multiple scopes + the sorted scope IDs are joined with ``"+"``. This is the synchronous + counterpart of :meth:`get_scopes`, used wherever a plain string key is + needed (e.g. scheduler priority queues). + """ + def get_scope_load(self, scope_id: str) -> float: """Return the current load of the scope identified by *scope_id*: its active sends divided by its concurrency limit (or by the global @@ -519,8 +530,22 @@ class ThrottlingManager: scopes = request.meta.get("throttling_scopes") if scopes is not None: return cast("RequestScopes", scopes) + download_slot = request.meta.get("download_slot") + if download_slot is not None: + warnings.warn( + "The 'download_slot' request meta key is deprecated. Use " + "'throttling_scopes' instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return download_slot return urlparse_cached(request).netloc + def get_slot_key(self, request: Request) -> str: + scopes = self._resolve_scopes_sync(request) + scope_ids = sorted(iter_scopes(scopes)) + return "+".join(scope_ids) if scope_ids else "" + def _cached_scope_values( self, request: Request ) -> list[tuple[ScopeID, float | None]]: @@ -1121,7 +1146,7 @@ class ThrottlingScopeManager: elif self._rampup_enabled: self._concurrency = self._min_concurrency else: - self._concurrency = None + self._concurrency = settings.getint("THROTTLING_SCOPE_CONCURRENCY") or None # Used as the load denominator when the scope enforces no explicit # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 90ed70262..fc86cbe7b 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,13 +66,13 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - # When needed, useful settings can be added here, e.g. ones that prevent - # deprecation warnings. settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } + if prevent_warnings: + settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index cb14b588d..19272a0a9 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,7 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy.core.downloader import Downloader, Slot, _get_concurrency_delay, tls +from scrapy.core.downloader import Downloader, Slot, _Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, _ScrapyClientContextFactory, @@ -41,33 +41,20 @@ if TYPE_CHECKING: class TestSlot: def test_repr(self): - slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - assert repr(slot) == "Slot(concurrency=8, delay=0.1, randomize_delay=True)" + slot = _Slot() + assert repr(slot) == "_Slot()" + def test_deprecated(self): + with pytest.warns(ScrapyDeprecationWarning, match="Slot class is deprecated"): + Slot() -class TestGetConcurrencyDelay: - def test_default(self): - crawler = get_crawler() - concurrency, delay = _get_concurrency_delay( - 8, DefaultSpider(), crawler.settings - ) - assert (concurrency, delay) == (8, 0.0) + def test_deprecated_subclass(self): + with pytest.warns( + ScrapyDeprecationWarning, match="inherits from the deprecated Slot" + ): - def test_spider_download_delay(self): - crawler = get_crawler() - spider = DefaultSpider() - spider.download_delay = 2.5 - _concurrency, delay = _get_concurrency_delay(8, spider, crawler.settings) - assert delay == 2.5 - - def test_download_delay_per_slot(self): - crawler = get_crawler(settings_dict={"DOWNLOAD_DELAY_PER_SLOT": 3.0}) - # DOWNLOAD_DELAY_PER_SLOT takes precedence over both DOWNLOAD_DELAY and - # the spider's download_delay attribute. - spider = DefaultSpider() - spider.download_delay = 2.5 - _concurrency, delay = _get_concurrency_delay(8, spider, crawler.settings) - assert delay == 3.0 + class MySlot(Slot): + pass @pytest.mark.requires_reactor # this test is related to the Twisted HTTP code diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0cddfd0ed..5588775cf 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio import logging import re +import warnings from pathlib import Path from typing import Any, ClassVar @@ -81,6 +82,60 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen + @pytest.mark.parametrize( + ("setting_name", "current_default", "future_default"), + [ + ("THROTTLING_SCOPE_CONCURRENCY", 8, 1), + ], + ) + def test_deprecated_default_settings_warn( + self, setting_name: str, current_default: Any, future_default: Any + ) -> None: + crawler = Crawler(DefaultSpider) + with pytest.warns( + ScrapyDeprecationWarning, + match=rf"The default value of {setting_name} will change from {current_default!r} to {future_default!r}", + ): + crawler._apply_settings() + + @pytest.mark.parametrize( + ("setting_name", "current_default"), + [ + ("THROTTLING_SCOPE_CONCURRENCY", 8), + ], + ) + def test_deprecated_default_settings_no_warn_when_set( + self, setting_name: str, current_default: int + ) -> None: + crawler = Crawler(DefaultSpider, {setting_name: current_default}) + with warnings.catch_warnings(): + warnings.simplefilter("error", ScrapyDeprecationWarning) + crawler._apply_settings() + + def test_spider_download_delay_deprecated(self) -> None: + class DelaySpider(DefaultSpider): + download_delay = 2.5 + + crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8}) + with pytest.warns( + ScrapyDeprecationWarning, match="'download_delay' spider attribute" + ): + crawler._apply_settings() + assert crawler.settings.getfloat("DOWNLOAD_DELAY") == 2.5 + + def test_spider_download_delay_overridden_by_setting(self) -> None: + class DelaySpider(DefaultSpider): + download_delay = 2.5 + + crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8}) + crawler.settings.set("DOWNLOAD_DELAY", 5.0, priority="spider") + with pytest.warns( + ScrapyDeprecationWarning, + match="'download_delay' spider attribute.*being ignored", + ): + crawler._apply_settings() + assert crawler.settings.getfloat("DOWNLOAD_DELAY") == 5.0 + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 9d58a6e09..5540c2e98 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -1,39 +1,32 @@ -import time from typing import Any +from urllib.parse import urlparse import pytest from scrapy import Request -from scrapy.core.downloader import Downloader, Slot +from scrapy.core.downloader import Downloader from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MetaSpider -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test class DownloaderSlotsSettingsTestSpider(MetaSpider): name = "downloader_slots" custom_settings = { - "DOWNLOAD_DELAY": 1, - "RANDOMIZE_DOWNLOAD_DELAY": False, "DOWNLOAD_SLOTS": { - "quotes.toscrape.com": { - "concurrency": 1, - "delay": 2, - "randomize_delay": False, - "throttle": False, - }, - "books.toscrape.com": {"delay": 3, "randomize_delay": False}, + "quotes.toscrape.com": {"concurrency": 1}, + "books.toscrape.com": {"concurrency": 2}, }, } def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) assert self.mockserver - self.default_slot = self.mockserver.host + self.default_slot = urlparse(self.mockserver.url("/")).netloc self.times: dict[str, list[float]] = {} async def start(self): @@ -45,65 +38,86 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider): def parse(self, response): slot = response.meta.get("download_slot", self.default_slot) - self.times[slot].append(time.time()) + self.times[slot].append(response.meta.get("download_latency")) url = self.mockserver.url(f"/?downloader_slot={slot}&req=2") yield Request(url, callback=self.not_parse, meta={"download_slot": slot}) def not_parse(self, response): slot = response.meta.get("download_slot", self.default_slot) - self.times[slot].append(time.time()) - - -class TestCrawl: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_delay(self): - crawler = get_crawler(DownloaderSlotsSettingsTestSpider) - yield crawler.crawl(mockserver=self.mockserver) - slots = crawler.engine.downloader.slots - times = crawler.spider.times - tolerance = 0.3 - - delays_real = {k: v[1] - v[0] for k, v in times.items()} - error_delta = { - k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) - for k, v in slots.items() - } - - assert max(list(error_delta.values())) < tolerance + self.times[slot].append(response.meta.get("download_latency")) @coroutine_test -async def test_params(): - params = { - "concurrency": 1, - "delay": 2, - "randomize_delay": False, - } - settings = { - "DOWNLOAD_SLOTS": { - "example.com": params, - }, - } +async def test_concurrency_key_deprecated(): + settings = {"DOWNLOAD_SLOTS": {"example.com": {"concurrency": 3}}} crawler = get_crawler(DefaultSpider, settings_dict=settings) crawler.spider = crawler._create_spider() + with pytest.warns(ScrapyDeprecationWarning) as warns: + downloader = Downloader(crawler) + messages = [str(w.message) for w in warns] + assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) + assert any("'concurrency' key in DOWNLOAD_SLOTS" in m for m in messages) + downloader._get_slot(Request("https://example.com")) + downloader.close() + + +@coroutine_test +async def test_download_slots_deprecated(): + settings = {"DOWNLOAD_SLOTS": {"example.com": {"concurrency": 2}}} + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + with pytest.warns( + ScrapyDeprecationWarning, match="DOWNLOAD_SLOTS setting is deprecated" + ): + Downloader(crawler).close() + + +@coroutine_test +async def test_slots_deprecated(): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() downloader = Downloader(crawler) request = Request("https://example.com") - _, actual = downloader._get_slot(request) + request.meta[Downloader.DOWNLOAD_SLOT] = "example.com" + downloader.active.add(request) + with pytest.warns(ScrapyDeprecationWarning, match="Downloader.slots is deprecated"): + slot = downloader.slots.get("example.com") + assert slot is not None + assert isinstance(slot.active, set) + assert request in slot.active + downloader.active.discard(request) + downloader.close() + + +@coroutine_test +async def test_download_slot_meta_deprecated(): + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + downloader = Downloader(crawler) + request = Request("https://example.com") + request.meta["download_slot"] = "custom" + with pytest.warns( + ScrapyDeprecationWarning, match="'download_slot' request meta key is deprecated" + ): + key, _ = downloader._get_slot(request) + downloader.close() + assert key == "custom" + + +@coroutine_test +async def test_delay_deprecated(): + settings = { + "DOWNLOAD_SLOTS": {"example.com": {"delay": 2, "randomize_delay": False}} + } + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + with pytest.warns(ScrapyDeprecationWarning) as warns: + downloader = Downloader(crawler) + messages = [str(w.message) for w in warns] + assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) + assert any("'delay' key in DOWNLOAD_SLOTS" in m for m in messages) + downloader._get_slot(Request("https://example.com")) downloader.close() - expected = Slot(**params) - for param in params: - assert getattr(expected, param) == getattr(actual, param), ( - f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}" - ) @coroutine_test @@ -122,7 +136,7 @@ async def test_get_slot_deprecated_spider_arg(): downloader.close() assert key1 == key2 - assert slot1 == slot2 + assert slot1._key == slot2._key @pytest.mark.parametrize( @@ -132,6 +146,7 @@ async def test_get_slot_deprecated_spider_arg(): "scrapy.pqueues.DownloaderAwarePriorityQueue", ], ) +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") @coroutine_test async def test_none_slot_with_priority_queue( mockserver: MockServer, priority_queue_class: str diff --git a/tests/test_engine.py b/tests/test_engine.py index 2b15c40d1..a7f81e19d 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -856,7 +856,7 @@ class TestEngineCloseSpider: engine = ExecutionEngine(crawler, lambda _: None) crawler.engine = engine await engine.open_spider_async() - del engine.downloader.slots + engine.downloader.close = Mock(side_effect=Exception("close failed")) await engine.close_spider_async() assert "Downloader close failure" in caplog.text diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index c45687a3f..cf7f94f04 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -4,7 +4,6 @@ from unittest.mock import Mock import pytest import queuelib -from scrapy.core.downloader import Downloader from scrapy.http.request import Request from scrapy.pqueues import ( DownloaderAwarePriorityQueue, @@ -192,22 +191,22 @@ class TestDownloaderAwarePriorityQueue: # No active downloads are tracked in the downloader, so every slot has # the same score and tie-breaking must not starve a slot. req_a1 = Request("https://example.org/a1") - req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a1.meta["throttling_scopes"] = "slot-a" req_b1 = Request("https://example.org/b1") - req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b1.meta["throttling_scopes"] = "slot-b" req_a2 = Request("https://example.org/a2") - req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a2.meta["throttling_scopes"] = "slot-a" req_b2 = Request("https://example.org/b2") - req_b2.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b2.meta["throttling_scopes"] = "slot-b" for request in (req_a1, req_b1, req_a2, req_b2): self.queue.push(request) slots = [ - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttling_scopes"], ] assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"] @@ -216,48 +215,49 @@ class TestDownloaderAwarePriorityQueue: # If the selected slot becomes empty, rotation should continue from # that slot marker to avoid restarting from the smallest slot. req_a1 = Request("https://example.org/a1") - req_a1.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a1.meta["throttling_scopes"] = "slot-a" req_a2 = Request("https://example.org/a2") - req_a2.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a2.meta["throttling_scopes"] = "slot-a" req_b1 = Request("https://example.org/b1") - req_b1.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b1.meta["throttling_scopes"] = "slot-b" req_c1 = Request("https://example.org/c1") - req_c1.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + req_c1.meta["throttling_scopes"] = "slot-c" for request in (req_a1, req_a2, req_b1, req_c1): self.queue.push(request) slots = [ - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], - self.queue.pop().meta[Downloader.DOWNLOAD_SLOT], + self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttling_scopes"], ] assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"] def test_pop_prefers_slot_with_fewer_active_downloads(self): - downloader = self.queue._downloader_interface.downloader + throttler = self.queue._downloader_interface._throttler + assert throttler is not None req_a = Request("https://example.org/a") - req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a" + req_a.meta["throttling_scopes"] = "slot-a" req_b = Request("https://example.org/b") - req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b" + req_b.meta["throttling_scopes"] = "slot-b" req_c = Request("https://example.org/c") - req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c" + req_c.meta["throttling_scopes"] = "slot-c" for req in (req_a, req_b, req_c): self.queue.push(req) - downloader.increment("slot-a") - downloader.increment("slot-c") + throttler._get_scope_manager("slot-a")._active = 1 + throttler._get_scope_manager("slot-c")._active = 1 popped = self.queue.pop() assert popped.url == req_b.url def test_contains(self): req = Request("https://example.org/") - req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot" + req.meta["throttling_scopes"] = "example-slot" assert "example-slot" not in self.queue self.queue.push(req) assert "example-slot" in self.queue diff --git a/tests/test_throttling.py b/tests/test_throttling.py index abcb29604..ef07bc381 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -570,8 +570,12 @@ class TestThrottlingScopeManager: scope.record_backoff(now=0.0) assert scope._delay == pytest.approx(3.0) - def test_no_scope_concurrency_limit_by_default(self): + def test_default_scope_concurrency(self): scope = _scope_manager() + assert scope._concurrency == 8 + + def test_no_scope_concurrency_limit_when_zero(self): + scope = _scope_manager(settings={"THROTTLING_SCOPE_CONCURRENCY": 0}) assert scope._concurrency is None for _ in range(100): scope.record_sent(now=0.0) From e0cc7ed7db3c93bb49c431a539824f83f75c59c3 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 11:33:37 +0200 Subject: [PATCH 22/55] WIP --- docs/topics/broad-crawls.rst | 2 +- docs/topics/throttling.rst | 13 +- scrapy/core/scheduler.py | 5 +- scrapy/crawler.py | 16 --- scrapy/extensions/throttle.py | 73 ++++++------ scrapy/settings/__init__.py | 25 ---- scrapy/settings/default_settings.py | 21 +--- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/throttling.py | 111 ++++++++++++++++-- tests/test_crawler.py | 35 +----- tests/test_settings/__init__.py | 9 -- tests/test_throttling.py | 6 +- 12 files changed, 162 insertions(+), 156 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index cace1f883..81a4679ba 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,7 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). +can be set per domain (:setting:`THROTTLING_SCOPE_CONCURRENCY`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 07b0fa0a5..fec88d265 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,9 +26,9 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN +- .. setting:: THROTTLING_SCOPE_CONCURRENCY - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` (:ref:`fallback `: ``8``)) + :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1`` (:ref:`fallback `: ``8``)) Maximum number of simultaneous requests per domain. @@ -49,7 +49,7 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps ``CONCURRENT_REQUESTS_PER_DOMAIN``. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, @@ -277,7 +277,7 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. Concurrency +:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). @@ -440,9 +440,8 @@ Its keys are scope names and its values are following keys: ``concurrency`` (:class:`int`) - Maximum number of concurrent requests for the scope. When unset, the - per-domain concurrency (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) applies - instead. + Maximum number of concurrent requests for the scope. When unset, + :setting:`THROTTLING_SCOPE_CONCURRENCY` applies instead. ``min_concurrency`` (:class:`int`) Concurrency floor that :ref:`backoff ` and :ref:`rampup ` diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 90a892dc3..f233f8c84 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -211,9 +211,8 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent - concurrently. + :setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLING_SCOPE_CONCURRENCY`, + those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired order. Lowering those settings to ``1`` enforces the desired order except diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 492877e7a..28bb2bf83 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -98,7 +98,6 @@ class Crawler: return self.addons.load_settings(self.settings) - self._warn_on_deprecated_default_settings() self._apply_spider_download_delay() self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -184,21 +183,6 @@ class Crawler: "DOWNLOAD_DELAY", spider.download_delay, priority="spider" ) - def _warn_on_deprecated_default_settings(self) -> None: - default_priority = SETTINGS_PRIORITIES["default"] - for setting_name, current_default, future_default in ( - ("THROTTLING_SCOPE_CONCURRENCY", 8, 1), - ): - if self.settings.getpriority(setting_name) == default_priority: - warnings.warn( - f"The default value of {setting_name} will change from " - f"{current_default!r} to {future_default!r} in a future " - f"Scrapy version. Explicitly set {setting_name} in your " - f"settings to silence this warning.", - category=ScrapyDeprecationWarning, - stacklevel=3, - ) - def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 42e58cbc9..d3203f335 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -6,14 +6,15 @@ from warnings import warn from scrapy import Request, Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy.core.downloader import Slot from scrapy.crawler import Crawler from scrapy.http import Response + from scrapy.throttling import ThrottlingManagerProtocol logger = logging.getLogger(__name__) @@ -43,6 +44,8 @@ class AutoThrottle: f"AUTOTHROTTLE_TARGET_CONCURRENCY " f"({self.target_concurrency!r}) must be higher than 0." ) + # Scopes whose start delay has already been applied (see _scope_delay). + self._started_scopes: set[str] = set() crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( self._response_downloaded, signal=signals.response_downloaded @@ -55,7 +58,9 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined] + self.startdelay = max( + self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") + ) def _min_delay(self, spider: Spider) -> float: s = self.crawler.settings @@ -64,55 +69,55 @@ class AutoThrottle: def _max_delay(self, spider: Spider) -> float: return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY") - def _start_delay(self, spider: Spider) -> float: - return max( - self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") - ) - def _response_downloaded( self, response: Response, request: Request, spider: Spider ) -> None: - key, slot = self._get_slot(request, spider) + throttler = self.crawler.throttler latency = request.meta.get("download_latency") if ( latency is None - or slot is None + or throttler is None or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return - olddelay = slot.delay - self._adjust_delay(slot, latency, response) + # AutoThrottle predates throttling scopes, so it adjusts the delay of + # the request's domain scope, matching its historical per-domain slots. + scope_id = urlparse_cached(request).netloc + olddelay = self._scope_delay(throttler, scope_id) + newdelay = self._adjust_delay(olddelay, latency, response) + throttler.set_scope_delay(scope_id, newdelay) if self.debug: - diff = slot.delay - olddelay - size = len(response.body) - conc = len(slot.transferring) logger.info( - "slot: %(slot)s | conc:%(concurrency)2d | " + "slot: %(slot)s | " "delay:%(delay)5d ms (%(delaydiff)+d) | " "latency:%(latency)5d ms | size:%(size)6d bytes", { - "slot": key, - "concurrency": conc, - "delay": slot.delay * 1000, - "delaydiff": diff * 1000, + "slot": scope_id, + "delay": newdelay * 1000, + "delaydiff": (newdelay - olddelay) * 1000, "latency": latency * 1000, - "size": size, + "size": len(response.body), }, extra={"spider": spider}, ) - def _get_slot( - self, request: Request, spider: Spider - ) -> tuple[str | None, Slot | None]: - key: str | None = request.meta.get("download_slot") - if key is None: - return None, None - assert self.crawler.engine - return key, self.crawler.engine.downloader.slots.get(key) + def _scope_delay( + self, throttler: ThrottlingManagerProtocol, scope_id: str + ) -> float: + """Return the current delay of *scope_id*, applying AUTOTHROTTLE_START_DELAY + the first time the scope is seen.""" + delay = throttler.get_scope_delay(scope_id) + if scope_id not in self._started_scopes: + self._started_scopes.add(scope_id) + delay = max(delay, self.startdelay) + return delay - def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None: - """Define delay adjustment policy""" + def _adjust_delay( + self, olddelay: float, latency: float, response: Response + ) -> float: + """Return the new delay given the current *olddelay* and the observed + *latency*.""" # If a server needs `latency` seconds to respond then # we should send a request each `latency/N` seconds @@ -120,7 +125,7 @@ class AutoThrottle: target_delay = latency / self.target_concurrency # Adjust the delay to make it closer to target_delay - new_delay = (slot.delay + target_delay) / 2.0 + new_delay = (olddelay + target_delay) / 2.0 # If target delay is bigger than old delay, then use it instead of mean. # It works better with problematic sites. @@ -133,7 +138,7 @@ class AutoThrottle: # than old one, as error pages (and redirections) are usually small and # so tend to reduce latency, thus provoking a positive feedback by # reducing delay instead of increase. - if response.status != 200 and new_delay <= slot.delay: - return + if response.status != 200 and new_delay <= olddelay: + return olddelay - slot.delay = new_delay + return new_delay diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 833c71f44..0ee6c83dd 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ - if name == "CONCURRENT_REQUESTS_PER_IP" and ( - isinstance(self[name], int) and self[name] != 0 - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " @@ -166,22 +157,6 @@ class BaseSettings(MutableMapping[str, Any]): stacklevel=2, ) - if name == "THROTTLING_SCOPE_CONCURRENCY": - per_domain_prio = self.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") or 0 - new_prio = self.getpriority(name) or 0 - if ( - per_domain_prio > SETTINGS_PRIORITIES["default"] - and new_prio <= SETTINGS_PRIORITIES["default"] - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " - "THROTTLING_SCOPE_CONCURRENCY instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - per_domain_val = self["CONCURRENT_REQUESTS_PER_DOMAIN"] - return per_domain_val if per_domain_val is not None else default - return self[name] if self[name] is not None else default def getbool(self, name: str, default: bool = False) -> bool: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 6bfd73777..6d8cdf154 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path -from typing import Any __all__ = [ "ADDONS", @@ -45,6 +44,7 @@ __all__ = [ "CONCURRENT_ITEMS", "CONCURRENT_REQUESTS", "CONCURRENT_REQUESTS_PER_DOMAIN", + "CONCURRENT_REQUESTS_PER_IP", "COOKIES_DEBUG", "COOKIES_ENABLED", "CRAWLSPIDER_FOLLOW_LINKS", @@ -265,6 +265,7 @@ CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 +CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False @@ -597,7 +598,7 @@ THROTTLING_SCOPES = {} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 -THROTTLING_SCOPE_CONCURRENCY = 8 +THROTTLING_SCOPE_CONCURRENCY = 1 THROTTLING_SCOPE_LIMIT = 100000 THROTTLING_SCOPE_MAX_IDLE = 3600.0 THROTTLING_DEBUG = False @@ -613,19 +614,3 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True - - -def __getattr__(name: str) -> Any: - if name == "CONCURRENT_REQUESTS_PER_IP": - import warnings # noqa: PLC0415 - - from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 - - warnings.warn( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return 0 - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 64b27db68..71c7596f5 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders" ROBOTSTXT_OBEY = True # Throttle crawls to be polite to websites: -CONCURRENT_REQUESTS_PER_DOMAIN = 1 +THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 # Set settings whose default value is deprecated to a future-proof value: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 040e15a28..fe69fe6b5 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -2,8 +2,10 @@ from __future__ import annotations import contextlib import datetime as dt +import ipaddress import logging import random +import re import time import warnings from collections import OrderedDict @@ -18,6 +20,7 @@ from typing_extensions import NotRequired, Self from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.resolver import dnscache from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object @@ -148,6 +151,59 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None +# A DNS hostname with at least one dot, e.g. "books.toscrape.com"; labels are +# alphanumeric (plus hyphens, not at the edges) and an optional trailing dot +# marks a fully-qualified name. +_HOSTNAME_RE = re.compile( + r"(?i)^(?!-)[a-z0-9-]{1,63}(? str: + """Classify *scope_id* as ``"ip"``, ``"domain"`` or ``"other"`` to pick its + default concurrency setting (see :func:`_default_scope_concurrency`). + + A scope that parses as an IP address is ``"ip"``; one that *looks like* a + DNS hostname with at least one dot is ``"domain"``; anything else (custom + group names, single labels like ``"localhost"``) is ``"other"``. + """ + host = scope_id + if host.startswith("[") and "]" in host: # bracketed IPv6, e.g. "[::1]:80" + host = host[1 : host.index("]")] + with contextlib.suppress(ValueError): + ipaddress.ip_address(host) + return "ip" + # Drop a trailing ":port" for the hostname check (an IPv4 "host:port" is + # re-tested as an IP below). + if host.count(":") == 1: + host = host.rsplit(":", 1)[0] + with contextlib.suppress(ValueError): + ipaddress.ip_address(host) + return "ip" + if "." in host and _HOSTNAME_RE.match(host): + return "domain" + return "other" + + +def _default_scope_concurrency(settings: Any, scope_id: ScopeID) -> int: + """Return the default concurrency for *scope_id* based on its + :func:`kind <_classify_scope>`. + + Domain scopes default to :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, IP + scopes to :setting:`CONCURRENT_REQUESTS_PER_IP` (falling back to the + per-domain value when per-IP limiting is off, so IP-literal crawls are not + left unbounded), and any other scope to :setting:`THROTTLING_SCOPE_CONCURRENCY`. + """ + kind = _classify_scope(scope_id) + if kind == "ip": + return settings.getint("CONCURRENT_REQUESTS_PER_IP") or settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + if kind == "domain": + return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + return settings.getint("THROTTLING_SCOPE_CONCURRENCY") + + def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]: """Normalize *collection* (``None``, str, iterable or dict) into a dict mapping scope names to values produced by *default*.""" @@ -398,6 +454,17 @@ class ThrottlingManagerProtocol(Protocol): whereas a ``delay_scope`` call is trusted. """ + def get_scope_delay(self, scope_id: str) -> float: + """Return the current base (non-backoff) delay of *scope_id*, in seconds.""" + + def set_scope_delay(self, scope_id: str, delay: float) -> None: + """Set the base (non-backoff) delay of *scope_id* to *delay* seconds. + + Unlike :meth:`delay_scope`, this both raises and lowers the delay and is + not counted as backoff; it lets a component drive the scope delay + directly (e.g. an adaptive-delay extension). + """ + async def process_response(self, response: Response) -> None: """Update the throttling state based on *response*.""" @@ -480,6 +547,9 @@ class ThrottlingManager: load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS") ) self._debug = crawler.settings.getbool("THROTTLING_DEBUG") + # When set, each request also gets an IP scope (its resolved address), + # enforced alongside its domain scope (see _resolve_scopes_sync). + self._per_ip: int = crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") self._robotstxt_obey = crawler.settings.getbool( "ROBOTSTXT_OBEY" @@ -539,7 +609,15 @@ class ThrottlingManager: stacklevel=2, ) return download_slot - return urlparse_cached(request).netloc + netloc = urlparse_cached(request).netloc + if self._per_ip: + # Best-effort, mirroring the legacy per-IP downloader slots: use the + # cached resolved address if available, else fall back to the domain + # scope alone (the IP is not known until the host is resolved). + ip = dnscache.get(netloc) + if isinstance(ip, str) and ip != netloc: + return (netloc, ip) + return netloc def get_slot_key(self, request: Request) -> str: scopes = self._resolve_scopes_sync(request) @@ -903,6 +981,14 @@ class ThrottlingManager: # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) + def get_scope_delay(self, scope_id: ScopeID) -> float: + return self._get_scope_manager(scope_id).get_base_delay() + + def set_scope_delay(self, scope_id: ScopeID, delay: float) -> None: + self._get_scope_manager(scope_id).set_base_delay( + float(delay), only_increase=False + ) + def _maybe_evict(self, now: float) -> None: if self._max_idle <= 0: return @@ -1002,11 +1088,15 @@ class ThrottlingScopeManagerProtocol(Protocol): reported for a request, correcting the estimate used by :meth:`record_sent`.""" - def set_base_delay(self, delay: float) -> None: - """Raise the base (non-backoff) delay of this scope to *delay* seconds. + def get_base_delay(self) -> float: + """Return the base (non-backoff) delay of this scope, in seconds.""" - It never lowers the configured base delay; it is used to honor external - hints such as a robots.txt ``Crawl-delay`` directive. + def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None: + """Set the base (non-backoff) delay of this scope to *delay* seconds. + + By default it only raises the delay, to honor external hints such as a + robots.txt ``Crawl-delay`` directive. Pass ``only_increase=False`` to + also allow lowering it. """ def set_concurrency(self, concurrency: int) -> None: @@ -1146,7 +1236,7 @@ class ThrottlingScopeManager: elif self._rampup_enabled: self._concurrency = self._min_concurrency else: - self._concurrency = settings.getint("THROTTLING_SCOPE_CONCURRENCY") or None + self._concurrency = _default_scope_concurrency(settings, self._id) or None # Used as the load denominator when the scope enforces no explicit # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") @@ -1374,10 +1464,15 @@ class ThrottlingScopeManager: elif consumed is not None: self._consumed = max(0.0, self._consumed + float(consumed)) - def set_base_delay(self, delay: float) -> None: - if delay <= self._base_delay: + def get_base_delay(self) -> float: + return self._base_delay + + def set_base_delay(self, delay: float, *, only_increase: bool = True) -> None: + if only_increase and delay <= self._base_delay: return self._base_delay = delay + # Reflect the change in the effective delay unless a backoff is raising + # it above the base right now. if self._backoff_level == 0: self._delay = delay diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 5588775cf..ada7e53f1 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio import logging import re -import warnings from pathlib import Path from typing import Any, ClassVar @@ -82,41 +81,11 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen - @pytest.mark.parametrize( - ("setting_name", "current_default", "future_default"), - [ - ("THROTTLING_SCOPE_CONCURRENCY", 8, 1), - ], - ) - def test_deprecated_default_settings_warn( - self, setting_name: str, current_default: Any, future_default: Any - ) -> None: - crawler = Crawler(DefaultSpider) - with pytest.warns( - ScrapyDeprecationWarning, - match=rf"The default value of {setting_name} will change from {current_default!r} to {future_default!r}", - ): - crawler._apply_settings() - - @pytest.mark.parametrize( - ("setting_name", "current_default"), - [ - ("THROTTLING_SCOPE_CONCURRENCY", 8), - ], - ) - def test_deprecated_default_settings_no_warn_when_set( - self, setting_name: str, current_default: int - ) -> None: - crawler = Crawler(DefaultSpider, {setting_name: current_default}) - with warnings.catch_warnings(): - warnings.simplefilter("error", ScrapyDeprecationWarning) - crawler._apply_settings() - def test_spider_download_delay_deprecated(self) -> None: class DelaySpider(DefaultSpider): download_delay = 2.5 - crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8}) + crawler = Crawler(DelaySpider) with pytest.warns( ScrapyDeprecationWarning, match="'download_delay' spider attribute" ): @@ -127,7 +96,7 @@ class TestCrawler(TestBaseCrawler): class DelaySpider(DefaultSpider): download_delay = 2.5 - crawler = Crawler(DelaySpider, {"THROTTLING_SCOPE_CONCURRENCY": 8}) + crawler = Crawler(DelaySpider) crawler.settings.set("DOWNLOAD_DELAY", 5.0, priority="spider") with pytest.warns( ScrapyDeprecationWarning, diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3282b0591..ab8a20c14 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -769,15 +769,6 @@ def test_deprecated_dns_resolver_setting(): settings.get("DNS_RESOLVER") -def test_deprecated_concurrent_requests_per_ip_setting(): - settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) - with pytest.warns( - ScrapyDeprecationWarning, - match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", - ): - settings.get("CONCURRENT_REQUESTS_PER_IP") - - class Component1: pass diff --git a/tests/test_throttling.py b/tests/test_throttling.py index ef07bc381..c9b28067f 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -575,7 +575,11 @@ class TestThrottlingScopeManager: assert scope._concurrency == 8 def test_no_scope_concurrency_limit_when_zero(self): - scope = _scope_manager(settings={"THROTTLING_SCOPE_CONCURRENCY": 0}) + # THROTTLING_SCOPE_CONCURRENCY governs scopes that are neither a domain + # nor an IP (here a bare "custom" group name). + scope = _scope_manager( + settings={"THROTTLING_SCOPE_CONCURRENCY": 0}, config={"id": "custom"} + ) assert scope._concurrency is None for _ in range(100): scope.record_sent(now=0.0) From ba8bb9e4e97ddeefb91fe26ca59c38cdfc63bb07 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 12:53:18 +0200 Subject: [PATCH 23/55] WIP --- docs/news.rst | 6 +-- docs/topics/settings.rst | 40 +++++++++++++++ docs/topics/throttling.rst | 80 ++++++++++++++++++++---------- scrapy/core/downloader/__init__.py | 2 +- scrapy/throttling.py | 53 +++++++++++++++----- scrapy/utils/test.py | 4 +- tests/test_throttling.py | 2 +- 7 files changed, 141 insertions(+), 46 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e5ccd561e..b923f8135 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3261,7 +3261,7 @@ New features - Settings corresponding to :setting:`DOWNLOAD_DELAY`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis - via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`) + via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`) - Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`) @@ -7448,7 +7448,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: - Item loaders now support nested loaders (:issue:`1467`). - ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`). - - Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved + - Added setting ``AUTOTHROTTLE_TARGET_CONCURRENCY`` and improved AutoThrottle docs (:issue:`1324`). - Added ``response.text`` to get body as unicode (:issue:`1730`). - Anonymous S3 connections (:issue:`1358`). @@ -8651,7 +8651,7 @@ Scrapy changes: - added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way - added options ``-o`` and ``-t`` to the :command:`runspider` command -- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` +- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with ``AUTOTHROTTLE_ENABLED`` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. - added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6275e2e56..d6b9c2623 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -882,6 +882,46 @@ Default: ``True`` Whether to enable downloader stats collection. +.. setting:: DOWNLOAD_BIND_ADDRESS + +DOWNLOAD_BIND_ADDRESS +--------------------- + +Default: ``None`` + +The default local outgoing address for download-handler connections. + +This setting can be either: + +- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local + port is chosen automatically, or + +- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a + specific local interface and a specific local port. + +For example: + +.. code-block:: python + + # Bind to this local address + DOWNLOAD_BIND_ADDRESS = "127.0.0.2" + +.. code-block:: python + + # Bind to this local address and local port + DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000) + +If set, built-in HTTP download handlers use this value by default. +Set the :reqmeta:`bindaddress` request meta key to override it for a specific +request. + +.. note:: + + Handling of this setting needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. Specifying the port is unsupported by + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + .. setting:: DOWNLOAD_HANDLERS DOWNLOAD_HANDLERS diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index fec88d265..13e9c0e50 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,9 +26,9 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: THROTTLING_SCOPE_CONCURRENCY +- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1`` (:ref:`fallback `: ``8``)) + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``8``) Maximum number of simultaneous requests per domain. @@ -49,7 +49,7 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, @@ -277,8 +277,8 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency -is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. +Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it @@ -440,8 +440,10 @@ Its keys are scope names and its values are following keys: ``concurrency`` (:class:`int`) - Maximum number of concurrent requests for the scope. When unset, - :setting:`THROTTLING_SCOPE_CONCURRENCY` applies instead. + Maximum number of concurrent requests for the scope. When unset, the + default depends on the kind of scope: :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` + for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and + :setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope. ``min_concurrency`` (:class:`int`) Concurrency floor that :ref:`backoff ` and :ref:`rampup ` @@ -452,7 +454,10 @@ following keys: :setting:`DOWNLOAD_DELAY`. ``jitter`` (:class:`float` or 2-:class:`list`) - Per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. + Magnitude of the random variation applied to ``delay``; the per-scope + override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. ``0`` disables it, ``0.5`` + means ±50% (the default when :setting:`RANDOMIZE_DOWNLOAD_DELAY` is on), and + a ``[low, high]`` pair multiplies the delay by ``1 + uniform(low, high)``. ``quota`` (:class:`float`) Maximum :ref:`quota ` consumed per ``window``. @@ -924,30 +929,39 @@ window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas Per-IP concurrency limiting --------------------------- -A concurrency limit keyed by IP is just a throttling scope whose id is the -request's IP, with a ``concurrency`` limit. A request then carries two scopes, -its domain and its IP, and is only sent when **both** allow it (see -:ref:`multiple-throttling-scopes`). +.. setting:: CONCURRENT_REQUESTS_PER_IP -- Implement a :ref:`throttling manager ` that adds - the request's IP as a second scope: +A concurrency limit keyed by IP is a throttling scope whose id is the request's +IP. A request then carries two scopes, its domain and its IP, and is only sent +when **both** allow it (see :ref:`multiple-throttling-scopes`). - .. code-block:: python +Set :setting:`CONCURRENT_REQUESTS_PER_IP` (default: ``0``, disabled) to a +positive number to enable this built in: the throttler adds the IP of each +request, resolved from the DNS cache, as a second scope, limited to that many +concurrent requests. IP scopes whose ``concurrency`` is left unset default to +:setting:`CONCURRENT_REQUESTS_PER_IP`. - import socket +For finer control (e.g. resolving IPs that are not in the DNS cache yet, or +grouping several hosts under one address), implement a :ref:`throttling manager +` that adds the request's IP as a second scope +instead: - from scrapy.throttling import ThrottlingManager, add_scope, scope_cache - from scrapy.utils.asyncio import run_in_thread - from scrapy.utils.httpobj import urlparse_cached +.. code-block:: python + + import socket + + from scrapy.throttling import ThrottlingManager, add_scope, scope_cache + from scrapy.utils.asyncio import run_in_thread + from scrapy.utils.httpobj import urlparse_cached - class IPThrottlingManager(ThrottlingManager): - @scope_cache - async def get_scopes(self, request): - scopes = await super().get_scopes(request) - host = urlparse_cached(request).hostname - address = await run_in_thread(socket.gethostbyname, host) - return add_scope(scopes, address) + class IPThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + host = urlparse_cached(request).hostname + address = await run_in_thread(socket.gethostbyname, host) + return add_scope(scopes, address) .. _throttling-settings: @@ -1033,6 +1047,18 @@ Additional settings Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. +- .. setting:: THROTTLING_SCOPE_CONCURRENCY + + :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) + + Default maximum number of concurrent requests for :ref:`throttling scopes + ` that are neither domains nor IPs, e.g. custom scopes + added by a :ref:`custom throttling manager `. + + Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + :setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set + in :setting:`THROTTLING_SCOPES` overrides this. + - .. setting:: THROTTLING_SCOPE_LIMIT :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) @@ -1068,7 +1094,7 @@ The ``AutoThrottle`` extension is deprecated in favor of the throttling and :ref:`backoff ` system described here, which is always active and does not need to be enabled. -Setting :setting:`AUTOTHROTTLE_ENABLED` to ``True`` still works but logs a +Setting ``AUTOTHROTTLE_ENABLED`` to ``True`` still works but logs a deprecation warning. To migrate, drop the ``AUTOTHROTTLE_*`` settings and use the following equivalents: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index e22b1192d..ec6ba527e 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -97,7 +97,7 @@ class _DeprecatedSlotView: @property def randomize_delay(self) -> bool: if self._scope is not None: - return self._scope._randomize # type: ignore[union-attr] + return bool(self._scope._jitter) # type: ignore[union-attr] return False @property diff --git a/scrapy/throttling.py b/scrapy/throttling.py index fe69fe6b5..2d7d14e15 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -92,6 +92,20 @@ class BackoffConfig(TypedDict, total=False): jitter: float | list[float] +class RampupConfig(TypedDict, total=False): + """Per-scope override of the rampup settings. + + Used as the value of the ``"rampup"`` key of :class:`ThrottlingScopeConfig` + entries when fine-tuning rampup beyond a plain ``True``. Any key left out + falls back to its default (or, for ``backoff_target``, to + :setting:`RAMPUP_BACKOFF_TARGET`). + """ + + backoff_target: float | list[float] + delay_factor: float + min_delay: float + + class ThrottlingScopeConfig(TypedDict, total=False): """Accepted keys of :setting:`THROTTLING_SCOPES` entries. @@ -105,10 +119,15 @@ class ThrottlingScopeConfig(TypedDict, total=False): """Floor. Never drop below this during backoff/rampup.""" delay: float + jitter: float | list[float] + """Magnitude of the random variation applied to ``delay``; the per-scope + override of :setting:`RANDOMIZE_DOWNLOAD_DELAY` (``0`` disables it, ``0.5`` + means ±50%).""" + quota: float window: float - rampup: bool + rampup: bool | RampupConfig manager: str | type """Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for @@ -116,6 +135,10 @@ class ThrottlingScopeConfig(TypedDict, total=False): backoff: BackoffConfig + ignore_robots_txt: bool + """Silence the warning logged when this configuration is more aggressive + than a robots.txt ``Crawl-delay``.""" + ScopeID = str BackoffData = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, BackoffScopeData] @@ -1199,8 +1222,11 @@ class ThrottlingScopeManager: self._base_delay: float = float( config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) ) - self._randomize: bool = bool( - config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")) + # Magnitude of the random variation applied to the (non-backoff) delay. + # Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay + # randomization is on, or to no variation when it is off. + self._jitter: float | list[float] = config.get( + "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 ) self._delay_factor: float = float( backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR")) @@ -1211,7 +1237,7 @@ class ThrottlingScopeManager: self._min_delay: float = float( backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY")) ) - self._jitter: float | list[float] = backoff.get( + self._backoff_jitter: float | list[float] = backoff.get( "jitter", settings.getfloat("BACKOFF_JITTER") ) self._window: float = settings.getfloat("BACKOFF_WINDOW") @@ -1272,17 +1298,18 @@ class ThrottlingScopeManager: return float(value[0]), float(value[1]) return float(value), float(value) - def _apply_jitter(self, value: float) -> float: - if isinstance(self._jitter, (list, tuple)): - low, high = self._jitter[0], self._jitter[1] + @staticmethod + def _apply_jitter(value: float, jitter: float | list[float]) -> float: + if isinstance(jitter, (list, tuple)): + low, high = jitter[0], jitter[1] return value * (1 + random.uniform(low, high)) # noqa: S311 - if not self._jitter: + if not jitter: return value - return value * random.uniform(1 - self._jitter, 1 + self._jitter) # noqa: S311 + return value * random.uniform(1 - jitter, 1 + jitter) # noqa: S311 def _effective_delay(self) -> float: - if self._backoff_level == 0 and self._randomize and self._delay > 0: - return random.uniform(0.5 * self._delay, 1.5 * self._delay) # noqa: S311 + if self._backoff_level == 0 and self._delay > 0: + return self._apply_jitter(self._delay, self._jitter) return self._delay def _recover(self, now: float) -> None: @@ -1447,7 +1474,9 @@ class ThrottlingScopeManager: ) grown = max(self._min_delay, grown) grown = min(grown, self._max_delay) - self._delay = min(self._apply_jitter(grown), self._max_delay) + self._delay = min( + self._apply_jitter(grown, self._backoff_jitter), self._max_delay + ) self._next_allowed_time = now + self._delay def reconcile_quota( diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fc86cbe7b..38bd8c124 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,13 +66,13 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings (see prevent_warnings). settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } - if prevent_warnings: - settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/test_throttling.py b/tests/test_throttling.py index c9b28067f..ddc14ff9b 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -1106,7 +1106,7 @@ class TestThrottlingScopeManagerEdges: config={"id": "x", "backoff": {"jitter": [0.0, 0.0]}, "min_delay": 1.0} ) # A [low, high] jitter range of [0, 0] leaves the value unchanged. - assert scope._apply_jitter(4.0) == pytest.approx(4.0) + assert scope._apply_jitter(4.0, scope._backoff_jitter) == pytest.approx(4.0) def test_effective_delay_randomized(self): scope = _scope_manager( From 3ae5ab40c1bd4cb668de04097dfb1bdff235315e Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 13:32:57 +0200 Subject: [PATCH 24/55] WIP --- docs/topics/api.rst | 2 + docs/topics/throttling.rst | 92 +++++-------------- scrapy/crawler.py | 6 ++ scrapy/settings/default_settings.py | 24 ++++- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/throttling.py | 2 +- 6 files changed, 58 insertions(+), 70 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 19082d9d7..edfc37959 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -93,6 +93,8 @@ how you :ref:`configure the downloader middlewares modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. + .. autoattribute:: throttler + .. attribute:: spider Spider currently being crawled. This is an instance of the spider class diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 13e9c0e50..d15557baa 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -74,14 +74,15 @@ behavior for specific domains [1]_. It is a dict that maps scope names to :class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. -Its default value allows faster crawling of the testing website using during +Its default value allows faster crawling of the testing websites used during the :ref:`tutorial ` while maintaining conservative defaults for other domains: .. code-block:: python THROTTLING_SCOPES = { - "quotes.toscrape.com": {"concurrency": 16, "delay": 0.0}, + "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, } Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are @@ -132,8 +133,7 @@ How backoff works ----------------- Every :ref:`throttling scope ` keeps a current delay that -starts at its configured value (``0`` by default, or :setting:`DOWNLOAD_DELAY` -for the default per-domain scopes). +starts at its configured ``"delay"`` (:setting:`DOWNLOAD_DELAY` by default). A **backoff trigger** is a response whose status code is in :setting:`BACKOFF_HTTP_CODES` or a download exception whose type is in @@ -533,10 +533,6 @@ method, define a :class:`dict` where keys are throttling scopes and values are :class:`float` values that indicate the expected quota consumption (it does not need to be exact). -Everything else being equal, :class:`~scrapy.pqueues.ScrapyPriorityQueue` will -prioritize requests that consume a higher portion of the available throttling -quota, to minimize the risk of those requests getting stuck. - .. _custom-throttling-scope-managers: Customizing throttling scope managers @@ -575,64 +571,22 @@ setting: }, } -A scope manager only needs to implement the -:class:`~scrapy.throttling.ThrottlingScopeManagerProtocol`. For example, the -following manager enforces a fixed quota per time window without any delay or -exponential backoff, similar to a fixed-window rate limiter: +Most custom scope managers subclass the default +:class:`~scrapy.throttling.ThrottlingScopeManager` and override only the methods +whose behavior they want to change; implementing the +:class:`~scrapy.throttling.ThrottlingScopeManagerProtocol` from scratch is also +supported. For example, this manager disables exponential :ref:`backoff +`, so a scope relies solely on its configured delay and quota: .. code-block:: python :caption: ``myproject/throttling.py`` - import time + from scrapy.throttling import ThrottlingScopeManager - class FixedWindowScopeManager: - def __init__(self, crawler, config): - self.limit = config["quota"] - self.window = config.get("window", 60.0) - self.reset_at = None - self.used = 0.0 - self.active = 0 - - @classmethod - def from_crawler(cls, crawler, config): - return cls(crawler, config) - - def can_send(self, now=None, amount=None): - now = now if now is not None else time.monotonic() - if self.reset_at is not None and now >= self.reset_at: - self.reset_at, self.used = None, 0.0 - if self.used and self.used + (amount or 0) > self.limit: - return self.reset_at - now - return 0.0 - - def record_sent(self, now=None, amount=None): - now = now if now is not None else time.monotonic() - if self.reset_at is None: - self.reset_at = now + self.window - self.used += amount or 0 - self.active += 1 - - def record_done(self, now=None): - self.active = max(0, self.active - 1) - - def record_backoff(self, delay=None, now=None): - pass # this manager does not back off - - def reconcile_quota(self, consumed=None, remaining=None, now=None): - if remaining is not None: - self.used = max(0.0, self.limit - remaining) - elif consumed is not None: - self.used = max(0.0, self.used + consumed) - - def set_base_delay(self, delay): - pass - - def set_concurrency(self, concurrency): - pass - - def is_idle(self, now, max_idle): - return self.active == 0 + class FixedWindowScopeManager(ThrottlingScopeManager): + def record_backoff(self, *args, **kwargs): + pass # never back off .. _throttling-aware-scheduler: @@ -685,12 +639,13 @@ Alternative approaches include: :caption: ``settings.py`` import tldextract + from scrapy.throttling import ThrottlingManager, scope_cache from scrapy.utils.httpobj import urlparse_cached - class MyThrottlingManager: - - def get_request_scopes(self, request): + class MyThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): extracted = tldextract.extract(request.url) if extracted.domain and extracted.suffix: return f"{extracted.domain}.{extracted.suffix}" @@ -715,12 +670,13 @@ Alternative approaches include: :caption: ``settings.py`` import tldextract + from scrapy.throttling import ThrottlingManager, scope_cache from scrapy.utils.httpobj import urlparse_cached - class MyThrottlingManager: - - def get_request_scopes(self, request): + class MyThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): extracted = tldextract.extract(request.url) if not (extracted.domain and extracted.suffix): return urlparse_cached(request).netloc @@ -870,7 +826,7 @@ Cost-capped throttling Imagine you are using an API that charges different requests differently, e.g. based on the features used, and you want to limit how much you spend per time -window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas +window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas ` for that: - Implement a :ref:`throttling manager ` that: @@ -1150,6 +1106,8 @@ API .. autoclass:: scrapy.throttling.BackoffConfig +.. autoclass:: scrapy.throttling.RampupConfig + .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope .. autofunction:: scrapy.throttling.update_scope_backoff diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 28bb2bf83..8d94f9cf4 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -85,6 +85,12 @@ class Crawler: self.logformatter: LogFormatter | None = None self.request_fingerprinter: RequestFingerprinterProtocol | None = None self.throttler: ThrottlingManagerProtocol | None = None + """The throttling manager of this crawler, an instance of + :setting:`THROTTLING_MANAGER`. + + It is ``None`` until the crawl starts. Components can use it to inspect + or drive :ref:`throttling ` at run time, e.g. through + :meth:`~scrapy.throttling.ThrottlingManagerProtocol.delay_scope`.""" self.spider: Spider | None = None self.engine: ExecutionEngine | None = None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 6d8cdf154..0b09dfd53 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -32,6 +32,13 @@ __all__ = [ "AWS_SESSION_TOKEN", "AWS_USE_SSL", "AWS_VERIFY", + "BACKOFF_DELAY_FACTOR", + "BACKOFF_EXCEPTIONS", + "BACKOFF_HTTP_CODES", + "BACKOFF_JITTER", + "BACKOFF_MAX_DELAY", + "BACKOFF_MIN_DELAY", + "BACKOFF_WINDOW", "BOT_NAME", "CLOSESPIDER_ERRORCOUNT", "CLOSESPIDER_ITEMCOUNT", @@ -51,6 +58,7 @@ __all__ = [ "DEFAULT_DROPITEM_LOG_LEVEL", "DEFAULT_ITEM_CLASS", "DEFAULT_REQUEST_HEADERS", + "DELAYED_REQUESTS_WARN_THRESHOLD", "DEPTH_LIMIT", "DEPTH_PRIORITY", "DEPTH_STATS_VERBOSE", @@ -167,6 +175,7 @@ __all__ = [ "PERIODIC_LOG_DELTA", "PERIODIC_LOG_STATS", "PERIODIC_LOG_TIMING_ENABLED", + "RAMPUP_BACKOFF_TARGET", "RANDOMIZE_DOWNLOAD_DELAY", "REACTOR_THREADPOOL_MAXSIZE", "REDIRECT_ENABLED", @@ -209,6 +218,16 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", + "THROTTLING_DEBUG", + "THROTTLING_MANAGER", + "THROTTLING_ROBOTSTXT_MAX_DELAY", + "THROTTLING_ROBOTSTXT_OBEY", + "THROTTLING_SCOPES", + "THROTTLING_SCOPE_CONCURRENCY", + "THROTTLING_SCOPE_LIMIT", + "THROTTLING_SCOPE_MANAGER", + "THROTTLING_SCOPE_MAX_IDLE", + "THROTTLING_WINDOW", "TWISTED_DNS_RESOLVER", "TWISTED_REACTOR", "TWISTED_REACTOR_ENABLED", @@ -594,7 +613,10 @@ TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager" THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager" -THROTTLING_SCOPES = {} +THROTTLING_SCOPES = { + "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, +} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 71c7596f5..64b27db68 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders" ROBOTSTXT_OBEY = True # Throttle crawls to be polite to websites: -THROTTLING_SCOPE_CONCURRENCY = 1 +CONCURRENT_REQUESTS_PER_DOMAIN = 1 DOWNLOAD_DELAY = 1 # Set settings whose default value is deprecated to a future-proof value: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 2d7d14e15..3d88c5ea6 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -528,7 +528,7 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: .. code-block:: python from scrapy.utils.httpobj import urlparse_cached - from scrapy.utils.throttling import scope_cache + from scrapy.throttling import scope_cache class MyThrottlingManager: From 0e7c89a6a2c8d9907d716feb85c2d9155c1dc0af Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 15:33:10 +0200 Subject: [PATCH 25/55] WIP --- docs/topics/throttling.rst | 8 +- scrapy/core/downloader/__init__.py | 18 ++- scrapy/settings/default_settings.py | 5 +- .../templates/project/module/settings.py.tmpl | 7 + scrapy/throttling.py | 145 ++++++++++++++---- 5 files changed, 140 insertions(+), 43 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index d15557baa..1f358f789 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -72,11 +72,11 @@ The :setting:`THROTTLING_SCOPES` setting allows you to customize throttling behavior for specific domains [1]_. It is a dict that maps scope names to -:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. +:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. It is empty by default. -Its default value allows faster crawling of the testing websites used during -the :ref:`tutorial ` while maintaining conservative defaults -for other domains: +:command:`startproject` scaffolds an entry for the testing websites used during +the :ref:`tutorial `, so that they are crawled faster while the +:ref:`conservative defaults ` still apply to other domains: .. code-block:: python diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ec6ba527e..50c2165eb 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,18 +86,18 @@ class _DeprecatedSlotView: @property def delay(self) -> float: if self._scope is not None: - return self._scope._delay # type: ignore[union-attr] + return self._scope.get_delay() return 0.0 @delay.setter def delay(self, value: float) -> None: if self._scope is not None: - self._scope._delay = value # type: ignore[union-attr] + self._scope.set_base_delay(value, only_increase=False) @property def randomize_delay(self) -> bool: if self._scope is not None: - return bool(self._scope._jitter) # type: ignore[union-attr] + return bool(self._scope.get_jitter()) return False @property @@ -109,11 +109,13 @@ class _DeprecatedSlotView: stacklevel=2, ) if self._scope is not None: - return self._scope._concurrency or 0 # type: ignore[union-attr] + return self._scope.get_concurrency() or 0 return 0 def free_transfer_slots(self) -> int: - concurrency = self._scope._concurrency if self._scope is not None else 0 # type: ignore[union-attr] + concurrency = ( + self._scope.get_concurrency() or 0 if self._scope is not None else 0 + ) return concurrency - len(self.transferring) def download_delay(self) -> float: @@ -132,7 +134,7 @@ class _DeprecatedSlotView: return f"_DeprecatedSlotView({self._key!r})" -class _DeprecatedSlotsView(Mapping): +class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): """Deprecated mapping view of active downloads, keyed by slot name.""" __slots__ = ("_downloader", "_throttler") @@ -152,7 +154,7 @@ class _DeprecatedSlotsView(Mapping): if key not in self._active_keys(): raise KeyError(key) scope = ( - self._throttler._get_scope_manager(key) + self._throttler.get_scope_manager(key) if self._throttler is not None else None ) @@ -238,7 +240,7 @@ class Downloader: ) -> tuple[str, _DeprecatedSlotView]: key = self._get_slot_key(request) scope = ( - self.crawler.throttler._get_scope_manager(key) + self.crawler.throttler.get_scope_manager(key) if self.crawler.throttler is not None else None ) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0b09dfd53..6dd6160b6 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -613,10 +613,7 @@ TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager" THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager" -THROTTLING_SCOPES = { - "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, - "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, -} +THROTTLING_SCOPES = {} THROTTLING_WINDOW = 60.0 THROTTLING_ROBOTSTXT_OBEY = True THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 64b27db68..592fa5619 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -16,5 +16,12 @@ ROBOTSTXT_OBEY = True CONCURRENT_REQUESTS_PER_DOMAIN = 1 DOWNLOAD_DELAY = 1 +# Crawl the tutorial websites faster, overriding the polite defaults above only +# for these domains (remove this once you crawl your own sites): +THROTTLING_SCOPES = { + "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, +} + # Set settings whose default value is deprecated to a future-proof value: FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 3d88c5ea6..737343b35 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -28,6 +28,7 @@ from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http import Request, Response + from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) @@ -208,7 +209,7 @@ def _classify_scope(scope_id: ScopeID) -> str: return "other" -def _default_scope_concurrency(settings: Any, scope_id: ScopeID) -> int: +def _default_scope_concurrency(settings: BaseSettings, scope_id: ScopeID) -> int: """Return the default concurrency for *scope_id* based on its :func:`kind <_classify_scope>`. @@ -348,16 +349,18 @@ class ThrottlingManagerProtocol(Protocol): those scopes are currently exhausted. - A dict with scope names as keys and dict values. Dict values - support the following keys: + support the following keys (matching :class:`BackoffScopeData`): - ``"delay"``: a float indicating how many seconds to wait before sending another request for the scope. - - ``"quota"``: a float indicating the remaining :ref:`throttling - quota `. + - ``"remaining"``: a float indicating the remaining + :ref:`throttling quota ` of the scope. - If ``"quota"`` is not specified, the resource is considered - exhausted. + - ``"consumed"``: a float indicating how much of the scope's + quota has already been consumed. + + An empty dict marks the scope as currently exhausted. For example: @@ -366,7 +369,7 @@ class ThrottlingManagerProtocol(Protocol): return { "scope1": {"delay": 5.0}, "scope2": {}, - "scope3": {"quota": 42.0}, + "scope3": {"remaining": 42.0}, } """ @@ -488,6 +491,10 @@ class ThrottlingManagerProtocol(Protocol): directly (e.g. an adaptive-delay extension). """ + def get_scope_manager(self, scope_id: str) -> ThrottlingScopeManagerProtocol: + """Return the :class:`ThrottlingScopeManagerProtocol` instance handling + the scope identified by *scope_id*, creating it if necessary.""" + async def process_response(self, response: Response) -> None: """Update the throttling state based on *response*.""" @@ -592,7 +599,28 @@ class ThrottlingManager: self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict( "THROTTLING_SCOPES" ) - # Ordered by least-recently-used first (see _get_scope_manager), so the + # Cheap pre-filter for the backoff methods: the union of the global + # triggers and every per-scope override. A response whose status (or an + # exception type) is not in the union cannot trigger backoff for any + # scope, so its scopes need not be resolved (see get_response_backoff + # and get_exception_backoff). Each scope still makes the final decision + # via its own triggers_backoff_* methods. + self._any_backoff_http_codes: set[int] = set(self._backoff_http_codes) + self._any_backoff_exceptions: tuple[type[BaseException], ...] = ( + self._backoff_exceptions + ) + for scope_config in self._scopes_config.values(): + backoff = scope_config.get("backoff") or {} + if "http_codes" in backoff: + self._any_backoff_http_codes.update( + int(code) for code in backoff["http_codes"] + ) + if "exceptions" in backoff: + self._any_backoff_exceptions += tuple( + load_object(exc) if isinstance(exc, str) else exc + for exc in backoff["exceptions"] + ) + # Ordered by least-recently-used first (see get_scope_manager), so the # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT). self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = ( OrderedDict() @@ -631,7 +659,7 @@ class ThrottlingManager: category=ScrapyDeprecationWarning, stacklevel=2, ) - return download_slot + return cast("RequestScopes", download_slot) netloc = urlparse_cached(request).netloc if self._per_ip: # Best-effort, mirroring the legacy per-IP downloader slots: use the @@ -686,12 +714,21 @@ class ThrottlingManager: assert response.request is not None if response.request.meta.get("throttling_dont_track"): return None - if response.status not in self._backoff_http_codes: + if response.status not in self._any_backoff_http_codes: return None scopes = await self._scopes(response.request) + matched = [ + scope + for scope in iter_scopes(scopes) + if self.get_scope_manager(scope).triggers_backoff_for_status( + response.status + ) + ] + if not matched: + return None if delay := self.get_response_delay(response): - scopes = {scope: {"delay": delay} for scope in iter_scopes(scopes)} - return scopes + return {scope: {"delay": delay} for scope in matched} + return matched def get_response_delay(self, response: Response) -> float | None: """Return the throttling delay requested by the response.""" @@ -711,13 +748,19 @@ class ThrottlingManager: ) -> BackoffData: if request.meta.get("throttling_dont_track"): return None - if isinstance(exception, self._backoff_exceptions): - return await self._scopes(request) - return None + if not isinstance(exception, self._any_backoff_exceptions): + return None + scopes = await self._scopes(request) + matched = [ + scope + for scope in iter_scopes(scopes) + if self.get_scope_manager(scope).triggers_backoff_for_exception(exception) + ] + return matched or None # -- Scope-state coordination (called from the request lifecycle) -------- - def _get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol: + def get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol: manager = self._scope_managers.get(scope_id) if manager is not None: # Mark as most-recently-used for the LRU scope limit. @@ -743,7 +786,7 @@ class ThrottlingManager: managers exceeds :setting:`THROTTLING_SCOPE_LIMIT` (``0`` disables the limit). - LRU order is kept by :meth:`_get_scope_manager` moving each accessed + LRU order is kept by :meth:`get_scope_manager` moving each accessed scope to the end, so the coldest scopes are at the front. Only scopes that are idle (no in-flight requests and no active backoff) are evicted; the just-created *keep* scope is never evicted. A scope evicted while @@ -771,7 +814,7 @@ class ThrottlingManager: if not scope_values: return managers = [ - (self._get_scope_manager(scope_id), value) + (self.get_scope_manager(scope_id), value) for scope_id, value in scope_values ] while True: @@ -817,7 +860,7 @@ class ThrottlingManager: if self._request_delay_deadline(request, now) > now: return False for scope_id, value in self._cached_scope_values(request): - manager = self._get_scope_manager(scope_id) + manager = self.get_scope_manager(scope_id) if manager.can_send(now=now, amount=value) > 0: return False if manager.concurrency_blocked(): @@ -831,7 +874,7 @@ class ThrottlingManager: # up on broad crawls. self._maybe_evict(time.monotonic()) managers = [ - (self._get_scope_manager(scope_id), value) + (self.get_scope_manager(scope_id), value) for scope_id, value in self._cached_scope_values(request) ] for manager, value in managers: @@ -842,12 +885,12 @@ class ThrottlingManager: now = time.monotonic() wait = max(0.0, self._request_delay_deadline(request, now) - now) for scope_id, value in self._cached_scope_values(request): - manager = self._get_scope_manager(scope_id) + manager = self.get_scope_manager(scope_id) wait = max(wait, manager.can_send(now=now, amount=value)) return wait if wait > 0 else None def get_scope_load(self, scope_id: ScopeID) -> float: - return self._get_scope_manager(scope_id).get_load() + return self.get_scope_manager(scope_id).get_load() def get_request_delay(self, request: Request, now: float | None = None) -> float: now = time.monotonic() if now is None else now @@ -921,7 +964,7 @@ class ThrottlingManager: else: items = ((scope_id, None) for scope_id in iter_scopes(data)) for scope_id, entry in items: - manager = self._get_scope_manager(scope_id) + manager = self.get_scope_manager(scope_id) delay = consumed = remaining = None if isinstance(entry, dict): delay = entry.get("delay") @@ -993,7 +1036,7 @@ class ThrottlingManager: return if self._debug: logger.debug(f"robots.txt Crawl-delay for scope {scope_id}: {capped}s") - manager = self._get_scope_manager(scope_id) + manager = self.get_scope_manager(scope_id) manager.set_base_delay(capped) manager.set_concurrency(1) @@ -1002,13 +1045,13 @@ class ThrottlingManager: logger.debug(f"Delaying scope {scope_id} for {delay:.2f}s") # Like a Retry-After / RateLimit-Reset header, this is a hard minimum # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. - self._get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) + self.get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) def get_scope_delay(self, scope_id: ScopeID) -> float: - return self._get_scope_manager(scope_id).get_base_delay() + return self.get_scope_manager(scope_id).get_base_delay() def set_scope_delay(self, scope_id: ScopeID, delay: float) -> None: - self._get_scope_manager(scope_id).set_base_delay( + self.get_scope_manager(scope_id).set_base_delay( float(delay), only_increase=False ) @@ -1111,6 +1154,26 @@ class ThrottlingScopeManagerProtocol(Protocol): reported for a request, correcting the estimate used by :meth:`record_sent`.""" + def triggers_backoff_for_status(self, status: int) -> bool: + """Return whether a response with the given *status* triggers backoff + for this scope (defaults to :setting:`BACKOFF_HTTP_CODES`).""" + + def triggers_backoff_for_exception(self, exception: BaseException) -> bool: + """Return whether *exception* triggers backoff for this scope (defaults + to :setting:`BACKOFF_EXCEPTIONS`).""" + + def get_delay(self) -> float: + """Return the current effective delay of this scope, in seconds, + including any active backoff (unlike :meth:`get_base_delay`).""" + + def get_jitter(self) -> float | list[float]: + """Return the magnitude of the random variation applied to the delay + (the per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`).""" + + def get_concurrency(self) -> int | None: + """Return the maximum number of concurrent requests allowed for this + scope, or ``None`` when the scope enforces no explicit limit.""" + def get_base_delay(self) -> float: """Return the base (non-backoff) delay of this scope, in seconds.""" @@ -1240,6 +1303,19 @@ class ThrottlingScopeManager: self._backoff_jitter: float | list[float] = backoff.get( "jitter", settings.getfloat("BACKOFF_JITTER") ) + # Which responses/exceptions trigger backoff for this scope. Each + # defaults to the matching global BACKOFF_* setting (see + # triggers_backoff_for_status / triggers_backoff_for_exception). + self._backoff_http_codes: set[int] = { + int(code) + for code in backoff.get( + "http_codes", settings.getlist("BACKOFF_HTTP_CODES") + ) + } + self._backoff_exceptions: tuple[type[BaseException], ...] = tuple( + load_object(exc) if isinstance(exc, str) else exc + for exc in backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) + ) self._window: float = settings.getfloat("BACKOFF_WINDOW") self._min_concurrency: int = int(config.get("min_concurrency", 1)) @@ -1493,6 +1569,21 @@ class ThrottlingScopeManager: elif consumed is not None: self._consumed = max(0.0, self._consumed + float(consumed)) + def triggers_backoff_for_status(self, status: int) -> bool: + return status in self._backoff_http_codes + + def triggers_backoff_for_exception(self, exception: BaseException) -> bool: + return isinstance(exception, self._backoff_exceptions) + + def get_delay(self) -> float: + return self._delay + + def get_jitter(self) -> float | list[float]: + return self._jitter + + def get_concurrency(self) -> int | None: + return self._concurrency + def get_base_delay(self) -> float: return self._base_delay From b3ea60354f594a56bf56ff88db5c8587f3e9b5ea Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 16:28:21 +0200 Subject: [PATCH 26/55] WIP --- docs/topics/throttling.rst | 10 +++++++++- scrapy/core/downloader/__init__.py | 10 ++++------ scrapy/pqueues.py | 5 ++++- scrapy/throttling.py | 2 +- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 1f358f789..ecb7f7a40 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -278,7 +278,8 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. -Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at +Concurrency is set to ``1`` and the delay is raised to at least the +``Crawl-Delay`` value (a larger configured delay is kept), capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it @@ -897,6 +898,13 @@ request, resolved from the DNS cache, as a second scope, limited to that many concurrent requests. IP scopes whose ``concurrency`` is left unset default to :setting:`CONCURRENT_REQUESTS_PER_IP`. +.. note:: Enabling :setting:`CONCURRENT_REQUESTS_PER_IP` requires the + :ref:`throttling-aware scheduler `. The default + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` does not support it + and raises an error at start up if it is set, so switch + :setting:`SCHEDULER` and :setting:`SCHEDULER_PRIORITY_QUEUE` as shown + there. + For finer control (e.g. resolving IPs that are not in the DNS cache yet, or grouping several hosts under one address), implement a :ref:`throttling manager ` that adds the request's IP as a second scope diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 50c2165eb..6941e35cc 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -12,7 +12,6 @@ from scrapy import Request, Spider, signals from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.resolver import dnscache from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import _defer_sleep_async, deferred_from_coro from scrapy.utils.deprecate import create_deprecated_class @@ -181,7 +180,6 @@ class Downloader: self._transferring: set[Request] = set() self.handlers: DownloadHandlers = DownloadHandlers(crawler) self.total_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS") - self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP") self.middleware: DownloaderMiddlewareManager = ( DownloaderMiddlewareManager.from_crawler(crawler) ) @@ -253,10 +251,10 @@ class Downloader: return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - key = urlparse_cached(request).netloc or "" - if self.ip_concurrency: - key = dnscache.get(key, key) - return key + # Per-IP grouping (CONCURRENT_REQUESTS_PER_IP) is now a throttling scope + # handled by the throttler, not a downloader slot key; this fallback (used + # only when no throttler is set) keys by domain alone. + return urlparse_cached(request).netloc or "" async def _enqueue_request(self, request: Request) -> Response: key = self._get_slot_key(request) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 4394f7407..211b0b256 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -339,7 +339,10 @@ class DownloaderAwarePriorityQueue: ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' + f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP. ' + "Set SCHEDULER_PRIORITY_QUEUE to " + "'scrapy.pqueues.ThrottlingAwarePriorityQueue' (along with the " + "matching SCHEDULER) to use per-IP concurrency limiting." ) if slot_startprios and not isinstance(slot_startprios, dict): diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 737343b35..0cfea5d6d 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1095,7 +1095,7 @@ class ThrottlingScopeManagerProtocol(Protocol): }, "rampup": { "backoff_target": 1, - "delay_factor": 0.8, + "delay_factor": 0.5, "min_delay": 0.05, }, } From f87717e05ad3928834f2f33ac730a4e6071d7947 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Tue, 30 Jun 2026 16:45:58 +0200 Subject: [PATCH 27/55] WIP --- docs/topics/throttling.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index ecb7f7a40..1c82a7c93 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -28,7 +28,8 @@ The main throttling :ref:`settings ` are: - .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``8``) + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` + (:ref:`fallback `: ``8``)) Maximum number of simultaneous requests per domain. From ce35abbce70632d1993ac86a31a6cd4b55b99932 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 09:14:10 +0200 Subject: [PATCH 28/55] WIP --- docs/topics/broad-crawls.rst | 2 +- docs/topics/request-response.rst | 2 ++ .../downloader/handlers/_base_streaming.py | 12 ---------- scrapy/core/downloader/handlers/http11.py | 8 +++++++ scrapy/throttling.py | 24 +++++++++++++++---- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 81a4679ba..cace1f883 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,7 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`THROTTLING_SCOPE_CONCURRENCY`). +can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index d7a76be17..0d8441900 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -732,6 +732,8 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`throttling_delay` +* :reqmeta:`throttling_dont_track` * :reqmeta:`throttling_scopes` * :reqmeta:`verbatim_url` diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py index b7aba8fd0..6742781e6 100644 --- a/scrapy/core/downloader/handlers/_base_streaming.py +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -84,19 +84,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon crawler.settings.get("DOWNLOAD_BIND_ADDRESS") ) self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") - # these are useful for many handlers but used in different ways by them self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") - scope_concurrencies = [ - scope["concurrency"] - for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() - if "concurrency" in scope - ] - self._pool_size_per_host: int = max( - [ - crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), - *scope_concurrencies, - ] - ) @staticmethod @abstractmethod diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f83c497bf..30b8d73e0 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,6 +92,12 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): from twisted.internet import reactor self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) + # Keep enough persistent connections per host to match the highest + # per-host concurrency the throttler may admit: the per-domain and + # per-IP limits, the default "other"-scope limit, and any explicit + # THROTTLING_SCOPES concurrency. Per-scope concurrency can grow beyond + # this at runtime (rampup); the excess simply uses non-persistent + # connections. scope_concurrencies = [ scope["concurrency"] for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() @@ -99,6 +105,8 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): ] self._pool.maxPersistentPerHost = max( [ + crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), + crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP"), crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), *scope_concurrencies, ] diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 0cfea5d6d..dcdfeb97d 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1025,6 +1025,14 @@ class ThrottlingManager: conflicts.append(f"delay={config['delay']!r} < Crawl-delay {capped}") if config.get("concurrency") is not None and int(config["concurrency"]) > 1: conflicts.append(f"concurrency={config['concurrency']!r} > 1") + # A min_concurrency floor above 1 keeps the scope above the single-slot + # concurrency that a Crawl-delay implies, since set_concurrency(1) is + # clamped back up to it. + if ( + config.get("min_concurrency") is not None + and int(config["min_concurrency"]) > 1 + ): + conflicts.append(f"min_concurrency={config['min_concurrency']!r} > 1") if conflicts: logger.warning( f"Throttling scope {scope_id!r} is configured with {' and '.join(conflicts)}, " @@ -1421,11 +1429,17 @@ class ThrottlingScopeManager: if self._rampup_window_start is None: self._rampup_window_start = now return - while now - self._rampup_window_start >= self._window: - self._rampup_window_start += self._window - if self._rampup_backoffs < self._rampup_target[0]: - self._rampup_step() - self._rampup_backoffs = 0 + if now - self._rampup_window_start < self._window: + return + # Catch up with elapsed time but apply at most one ramp step per call: a + # scope that stayed idle for several windows must not ramp up + # cumulatively once it becomes active again (that would collapse the + # delay or jump the concurrency limit by several steps at once). + elapsed_windows = int((now - self._rampup_window_start) // self._window) + self._rampup_window_start += elapsed_windows * self._window + if self._rampup_backoffs < self._rampup_target[0]: + self._rampup_step() + self._rampup_backoffs = 0 def _rampup_step(self) -> None: # Backoff in progress: let it recover before probing again. From 074f9e567dfa86bf647b695b4725e48f59366a66 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 10:19:27 +0200 Subject: [PATCH 29/55] WIP --- docs/conf.py | 1 - docs/topics/downloader-middleware.rst | 22 ++ docs/topics/throttling.rst | 87 ++++-- scrapy/core/engine.py | 15 +- scrapy/downloadermiddlewares/backoff.py | 169 ++++++++++ scrapy/settings/default_settings.py | 1 + scrapy/throttling.py | 397 +++++++----------------- 7 files changed, 357 insertions(+), 335 deletions(-) create mode 100644 scrapy/downloadermiddlewares/backoff.py diff --git a/docs/conf.py b/docs/conf.py index 2fc7afe8c..7770e550b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,7 +144,6 @@ coverage_ignore_pyobjects = [ # -- Options for the autodoc extension ---------------------------------------- autodoc_type_aliases = { - "BackoffData": "BackoffData", "RequestScopes": "RequestScopes", } diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index b2259eacb..83f28cf2a 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -169,6 +169,28 @@ middleware, see the :ref:`downloader middleware usage guide For a list of the components enabled by default (and their orders) see the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting. +.. _backoff-mw: + +BackoffMiddleware +----------------- + +.. module:: scrapy.downloadermiddlewares.backoff + :synopsis: Backoff Downloader Middleware + +.. class:: BackoffMiddleware + + This middleware feeds download outcomes into :ref:`throttling `: + for every response matching :setting:`BACKOFF_HTTP_CODES` and every download + exception matching :setting:`BACKOFF_EXCEPTIONS`, it makes the request's + :ref:`throttling scopes ` :ref:`back off `. + + It runs below + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` so that it + observes rate-limiting responses (``429``, ``503``, …) before they are + turned into retries. + + See :ref:`throttling` for details. + .. _cookies-mw: CookiesMiddleware diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 1c82a7c93..09b8e4113 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -165,6 +165,14 @@ full speed gradually once it recovers. To keep a scope hovering around a target rate instead of repeatedly probing and backing off, enable :ref:`rampup `. +Backoff triggers are detected by the +:class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, a built-in +:ref:`downloader middleware ` enabled by default. +Any component can also trigger backoff programmatically for arbitrary scopes — +e.g. based on the response body of a specific site — through +:meth:`crawler.throttler.back_off() +`. + .. _per-scope-backoff: Per-scope backoff configuration @@ -793,32 +801,43 @@ to: target_domain = urlparse(target_url).netloc return add_scope(scopes, target_domain) - - Can differentiate between exhaustion of the target website and - exhaustion of the API itself. For example: +- Add a :ref:`downloader middleware ` that + differentiates between exhaustion of the target website and exhaustion of + the API itself. The API returns ``200`` even when the target website + rate-limits it, reporting the upstream status in a header; the middleware + backs off the **target-website** scope (not the API scope) in that case, + reusing the scope's own :meth:`~scrapy.throttling.ThrottlingScopeManagerProtocol.triggers_backoff_for_status` + check: - .. code-block:: python + .. code-block:: python - from scrapy.throttling import ThrottlingManager - from scrapy.utils.httpobj import urlparse_cached + from scrapy.throttling import iter_scopes + from scrapy.utils.httpobj import urlparse_cached - class MyThrottlingManager(ThrottlingManager): - async def get_response_backoff(self, response): - if ( - urlparse_cached(response.request).netloc != "api.toscrape.com" - or response.status != 200 - ): - return await super().get_response_backoff(response) - upstream_status_code = int( + class UpstreamBackoffMiddleware: + def __init__(self, crawler): + self.throttler = crawler.throttler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def process_response(self, request, response, spider): + if urlparse_cached(request).netloc == "api.toscrape.com": + upstream_status = int( response.headers.get("X-Upstream-Status-Code", b"200") ) - upstream_response = response.__class__( - response.url, - status=upstream_status_code, - headers=response.headers, - body=response.body, - ) - return await super().get_response_backoff(upstream_response) + scopes = [ + scope + for scope in iter_scopes(self.throttler.get_resolved_scopes(request)) + if scope != "api.toscrape.com" + and self.throttler.get_scope_manager(scope).triggers_backoff_for_status( + upstream_status + ) + ] + self.throttler.back_off(scopes) + return response .. _cost-smoothing-throttling: @@ -851,23 +870,27 @@ window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas return scopes return add_scope(scopes, "cost", estimate_request_cost(request)) - - Reconciles the estimated cost with the actual cost reported by the - response, so that the quota tracks real spending: +- Add a :ref:`downloader middleware ` that + reconciles the estimated cost with the actual cost reported by the + response, so that the quota tracks real spending: - .. code-block:: python + .. code-block:: python - from scrapy.throttling import ThrottlingManager, update_scope_backoff + class CostReconcileMiddleware: + def __init__(self, crawler): + self.throttler = crawler.throttler + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) - class MyThrottlingManager(ThrottlingManager): - async def get_response_backoff(self, response): - backoff = await super().get_response_backoff(response) - if response.headers.get("X-Actual-Cost") is None: - return backoff - estimated = estimate_request_cost(response.request) + def process_response(self, request, response, spider): + if response.headers.get("X-Actual-Cost") is not None: + estimated = estimate_request_cost(request) actual = float(response.headers[b"X-Actual-Cost"]) # Report the difference between actual and estimated cost. - return update_scope_backoff(backoff, "cost", consumed=actual - estimated) + self.throttler.reconcile_quota("cost", consumed=actual - estimated) + return response - Use the :setting:`THROTTLING_SCOPES` setting to set a maximum cost per time window: @@ -1101,7 +1124,6 @@ API :member-order: bysource .. autoclass:: scrapy.throttling.ThrottlingManager - :members: get_response_delay .. autoclass:: scrapy.throttling.ThrottlingScopeManagerProtocol :members: @@ -1119,4 +1141,3 @@ API .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope -.. autofunction:: scrapy.throttling.update_scope_backoff diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 75a70ce9a..b18ebf0d0 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -627,15 +627,11 @@ class ExecutionEngine: assert throttler is not None try: yield self._acquire_throttling(request) - try: - result: Response | Request - if self._downloader_fetch_needs_spider: - result = yield self.downloader.fetch(request, self.spider) - else: - result = yield self.downloader.fetch(request) - except Exception as exc: - yield deferred_from_coro(throttler.process_exception(request, exc)) - raise + result: Response | Request + if self._downloader_fetch_needs_spider: + result = yield self.downloader.fetch(request, self.spider) + else: + result = yield self.downloader.fetch(request) if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -643,7 +639,6 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request - yield deferred_from_coro(throttler.process_response(result)) logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: logger.log( diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py new file mode 100644 index 000000000..173ef1e62 --- /dev/null +++ b/scrapy/downloadermiddlewares/backoff.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import datetime as dt +import logging +from email.utils import parsedate_to_datetime +from typing import TYPE_CHECKING + +from scrapy.throttling import iter_scopes +from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.misc import load_object + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + import scrapy + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + from scrapy.throttling import ThrottlingManagerProtocol + + +logger = logging.getLogger(__name__) + + +def _parse_retry_after(response: Response) -> float | None: + raw = response.headers.get("Retry-After") + if not raw: + return None + try: + value = raw.decode("utf-8").strip() + except UnicodeDecodeError: + return None + if value.isdigit(): + return float(value) # seconds + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + return None + if date.tzinfo is None: + date = date.replace(tzinfo=dt.timezone.utc) + now = dt.datetime.now(dt.timezone.utc) + seconds_to_wait = (date - now).total_seconds() + # Keep sub-second precision (a date less than a second away must not be + # truncated to 0 and dropped); a past or present date yields no delay. + return max(0.0, seconds_to_wait) or None + + +def _parse_ratelimit_reset(response: Response) -> float | None: + raw = response.headers.get("RateLimit-Reset") + if not raw: + return None + try: + value = raw.decode("utf-8").strip() + except UnicodeDecodeError: + return None + try: + return float(value) + except ValueError: + return None + + +class BackoffMiddleware: + """Downloader middleware that drives :ref:`backoff ` from download + outcomes. + + It observes every response and download exception and, for those matching + :setting:`BACKOFF_HTTP_CODES` or :setting:`BACKOFF_EXCEPTIONS` (globally or + per :setting:`THROTTLING_SCOPES` scope), tells the :ref:`throttling manager + ` to back off the request's scopes through its + :meth:`~scrapy.throttling.ThrottlingManagerProtocol.back_off` API. + + It sits below :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` + in :setting:`DOWNLOADER_MIDDLEWARES_BASE` so it sees rate-limiting responses + (429, 503, …) before the retry middleware turns them into new requests. + """ + + def __init__(self, crawler: Crawler): + # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a + # non-None default and is instantiated before the downloader is built, + # so crawler.throttler is always set here (the engine likewise asserts + # it in its download path). + assert crawler.throttler is not None + self._throttler: ThrottlingManagerProtocol = crawler.throttler + settings = crawler.settings + # Union of the global backoff triggers and every per-scope override: a + # response status (or exception type) outside it cannot trigger backoff + # for any scope, so the scopes of such a request need not be resolved. + # Each scope still makes the final decision via its scope manager's + # triggers_backoff_* methods (which read the per-scope overrides). + self._http_codes: set[int] = { + int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") + } + self._exceptions: tuple[type[BaseException], ...] = tuple( + load_object(exc) if isinstance(exc, str) else exc + for exc in settings.getlist("BACKOFF_EXCEPTIONS") + ) + for scope_config in settings.getdict("THROTTLING_SCOPES").values(): + backoff = scope_config.get("backoff") or {} + if "http_codes" in backoff: + self._http_codes.update(int(code) for code in backoff["http_codes"]) + if "exceptions" in backoff: + self._exceptions += tuple( + load_object(exc) if isinstance(exc, str) else exc + for exc in backoff["exceptions"] + ) + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + @_warn_spider_arg + def process_response( + self, + request: Request, + response: Response, + spider: scrapy.Spider | None = None, + ) -> Response: + if ( + response.status not in self._http_codes + or "cached" in response.flags + or request.meta.get("throttling_dont_track") + ): + return response + matched = [ + scope + for scope in iter_scopes(self._throttler.get_resolved_scopes(request)) + if self._throttler.get_scope_manager(scope).triggers_backoff_for_status( + response.status + ) + ] + if matched: + self._throttler.back_off(matched, delay=self._response_delay(response)) + return response + + @_warn_spider_arg + def process_exception( + self, + request: Request, + exception: Exception, + spider: scrapy.Spider | None = None, + ) -> None: + if request.meta.get("throttling_dont_track") or not isinstance( + exception, self._exceptions + ): + return + matched = [ + scope + for scope in iter_scopes(self._throttler.get_resolved_scopes(request)) + if self._throttler.get_scope_manager(scope).triggers_backoff_for_exception( + exception + ) + ] + if matched: + self._throttler.back_off(matched) + return + + @staticmethod + def _response_delay(response: Response) -> float | None: + """Return the hard minimum delay requested by *response* through a + ``Retry-After`` or ``RateLimit-Reset`` header, or ``None``.""" + delays = [ + delay + for delay in ( + _parse_retry_after(response), + _parse_ratelimit_reset(response), + ) + if delay is not None + ] + return max(delays) if delays else None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 6dd6160b6..f6e677e8b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -359,6 +359,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, + "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 630, "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, diff --git a/scrapy/throttling.py b/scrapy/throttling.py index dcdfeb97d..5b4079746 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1,7 +1,6 @@ from __future__ import annotations import contextlib -import datetime as dt import ipaddress import logging import random @@ -10,13 +9,12 @@ import time import warnings from collections import OrderedDict from collections.abc import Awaitable, Callable, Iterable -from email.utils import parsedate_to_datetime from functools import wraps from typing import TYPE_CHECKING, Any, Protocol, TypedDict, TypeVar, cast from weakref import WeakKeyDictionary from twisted.internet.defer import Deferred -from typing_extensions import NotRequired, Self +from typing_extensions import Self from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning @@ -27,56 +25,13 @@ from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler - from scrapy.http import Request, Response + from scrapy.http import Request from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) -def _parse_retry_after(response: Response) -> float | None: - raw = response.headers.get("Retry-After") - if not raw: - return None - try: - value = raw.decode("utf-8").strip() - except UnicodeDecodeError: - return None - if value.isdigit(): - return float(value) # seconds - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError, OverflowError): - return None - if date.tzinfo is None: - date = date.replace(tzinfo=dt.timezone.utc) - now = dt.datetime.now(dt.timezone.utc) - seconds_to_wait = (date - now).total_seconds() - # Keep sub-second precision (a date less than a second away must not be - # truncated to 0 and dropped); a past or present date yields no delay. - return max(0.0, seconds_to_wait) or None - - -def _parse_ratelimit_reset(response: Response) -> float | None: - raw = response.headers.get("RateLimit-Reset") - if not raw: - return None - try: - value = raw.decode("utf-8").strip() - except UnicodeDecodeError: - return None - try: - return float(value) - except ValueError: - return None - - -class BackoffScopeData(TypedDict): - delay: NotRequired[float] - consumed: NotRequired[float] - remaining: NotRequired[float] - - class BackoffConfig(TypedDict, total=False): """Per-scope override of the backoff settings. @@ -142,7 +97,6 @@ class ThrottlingScopeConfig(TypedDict, total=False): ScopeID = str -BackoffData = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, BackoffScopeData] RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None] @@ -285,36 +239,6 @@ def add_scope( return scopes -def update_scope_backoff( - backoff: BackoffData, - scope: ScopeID, - /, - *, - delay: float | None = None, - consumed: float | None = None, -) -> BackoffData: - """Add *scope* to *backoff* or update its existing entry with the given - parameters. - - This is a utility function to help extending the output of - :meth:`~ThrottlingManagerProtocol.get_initial_backoff`, - :meth:`~ThrottlingManagerProtocol.get_response_backoff` or - :meth:`~ThrottlingManagerProtocol.get_exception_backoff`, e.g. in - :class:`ThrottlingManager` subclasses. - """ - if delay is None and consumed is None: - return cast("BackoffData", _add_bare_scope(backoff, scope, {})) - backoff = _to_scope_dict(backoff, dict) - entry = backoff.setdefault(scope, {}) - if not isinstance(entry, dict): - raise TypeError(f"Scope {scope!r} has a non-dict value in {backoff!r}") - if delay is not None: - entry["delay"] = delay - if consumed is not None: - entry["consumed"] = consumed - return backoff - - class ThrottlingManagerProtocol(Protocol): """A protocol for :setting:`THROTTLING_MANAGER` :ref:`components `.""" @@ -328,64 +252,17 @@ class ThrottlingManagerProtocol(Protocol): keys and :ref:`throttling quotas ` as values. """ - async def get_initial_backoff(self) -> BackoffData: - """Return the initial throttling data. + def get_resolved_scopes(self, request: Request) -> RequestScopes: + """Return the :ref:`throttling scopes ` under which + *request* was (or will be) sent, without re-resolving them. - This method is called before the first request is sent, and it should - be used to provide an initial throttling state, to be used before it is - updated with later calls to :meth:`get_response_backoff` and - :meth:`get_exception_backoff`. - - **Return values:** - - You may return any of the following: - - - ``None``: no throttling data to report. - - - A string: a single scope name, indicating that the scope is - currently exhausted. - - - An iterable of strings: multiple scope names, indicating that - those scopes are currently exhausted. - - - A dict with scope names as keys and dict values. Dict values - support the following keys (matching :class:`BackoffScopeData`): - - - ``"delay"``: a float indicating how many seconds to wait before - sending another request for the scope. - - - ``"remaining"``: a float indicating the remaining - :ref:`throttling quota ` of the scope. - - - ``"consumed"``: a float indicating how much of the scope's - quota has already been consumed. - - An empty dict marks the scope as currently exhausted. - - For example: - - .. code-block:: python - - return { - "scope1": {"delay": 5.0}, - "scope2": {}, - "scope3": {"remaining": 42.0}, - } - """ - - async def get_response_backoff(self, response: Response) -> BackoffData: - """Return a throttling data update based on *response*. - - It supports the same return values as :meth:`get_initial_backoff`. - """ - - async def get_exception_backoff( - self, request: Request, exception: Exception - ) -> BackoffData: - """Return a throttling data update based on *exception* and the - *request* that caused it. - - It supports the same return values as :meth:`get_initial_backoff`. + This is the synchronous counterpart of :meth:`get_scopes`: it returns + the scopes resolved earlier (e.g. at enqueue or :meth:`acquire` time) + and persisted on ``request.meta``, falling back to a best-effort + synchronous resolution only if none were persisted. Use it, rather than + :meth:`get_scopes`, to attribute a response or exception to the very + scopes the request was sent under — e.g. from a downloader middleware or + a spider callback that wants to :meth:`back_off` based on the response. """ async def acquire(self, request: Request) -> None: @@ -467,17 +344,62 @@ class ThrottlingManagerProtocol(Protocol): that share its scopes. """ + def back_off( + self, + scopes: RequestScopes, + *, + delay: float | None = None, + cap: bool = True, + ) -> None: + """Register a :ref:`backoff ` trigger for each of *scopes*. + + This is the general-purpose way to make a scope slow down, available to + any component through :attr:`crawler.throttler + `. The built-in :class:`backoff + middleware ` + calls it for :setting:`BACKOFF_HTTP_CODES` responses and + :setting:`BACKOFF_EXCEPTIONS` exceptions, but a downloader middleware or + spider callback can call it too (e.g. to back off based on the response + body of a specific site). + + *scopes* accepts the same shapes as the output of :meth:`get_scopes` + (typically the result of :meth:`get_resolved_scopes` for a request). + + When *delay* is ``None`` an exponential backoff step is applied; when + given, *delay* is a hard minimum delay in seconds (e.g. from a + :ref:`Retry-After ` header). *cap* limits *delay* to + :setting:`BACKOFF_MAX_DELAY`; set it to ``False`` for trusted, + programmatic delays. + """ + def delay_scope(self, scope_id: str, delay: float) -> None: """Hold back every request of the scope identified by *scope_id* for at least *delay* seconds, counted as a :ref:`backoff ` trigger for the scope. - This is the programmatic equivalent of a :ref:`Retry-After - ` response header, available to any component through - :attr:`crawler.throttler `. Unlike - those headers, *delay* is **not** capped at - :setting:`BACKOFF_MAX_DELAY`: that cap guards against untrusted input, - whereas a ``delay_scope`` call is trusted. + This is shorthand for :meth:`back_off(scope_id, delay=delay, + cap=False) `: the programmatic equivalent of a + :ref:`Retry-After ` response header. Unlike those headers, + *delay* is **not** capped at :setting:`BACKOFF_MAX_DELAY`: that cap + guards against untrusted input, whereas a ``delay_scope`` call is + trusted. + """ + + def reconcile_quota( + self, + scopes: RequestScopes, + *, + consumed: float | None = None, + remaining: float | None = None, + ) -> None: + """Reconcile the :ref:`throttling quota ` of each of + *scopes* with an actually *consumed* amount (a delta to add) or a + *remaining* amount (an absolute value), correcting the estimate used + when requests were sent. + + Like :meth:`back_off`, this is meant to be called from a downloader + middleware or spider callback that learns the real quota cost of a + request from its response. """ def get_scope_delay(self, scope_id: str) -> float: @@ -495,12 +417,6 @@ class ThrottlingManagerProtocol(Protocol): """Return the :class:`ThrottlingScopeManagerProtocol` instance handling the scope identified by *scope_id*, creating it if necessary.""" - async def process_response(self, response: Response) -> None: - """Update the throttling state based on *response*.""" - - async def process_exception(self, request: Request, exception: Exception) -> None: - """Update the throttling state based on a download *exception*.""" - _GetScopesMethod = TypeVar( "_GetScopesMethod", bound=Callable[..., Awaitable[RequestScopes]] @@ -517,9 +433,10 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: implementations that persists the resolved scopes on ``request.meta``. The readers of the resolved scopes — the synchronous readiness API of a - :ref:`throttling-aware scheduler ` and the - backoff methods — read this persisted value instead of resolving the scopes - again, so they stay cheap and consistent, and it survives a request being + :ref:`throttling-aware scheduler ` and + :meth:`~ThrottlingManagerProtocol.get_resolved_scopes` — read this persisted + value instead of resolving the scopes again, so they stay cheap and + consistent, and it survives a request being serialized to and restored from a :ref:`disk queue ` (which is what lets the readiness API resolve the scopes of a restored request synchronously). @@ -570,12 +487,6 @@ class ThrottlingManager: def __init__(self, crawler: Crawler) -> None: self.crawler = crawler - self._backoff_http_codes = { - int(code) for code in crawler.settings.getlist("BACKOFF_HTTP_CODES") - } - self._backoff_exceptions = tuple( - load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS") - ) self._debug = crawler.settings.getbool("THROTTLING_DEBUG") # When set, each request also gets an IP scope (its resolved address), # enforced alongside its domain scope (see _resolve_scopes_sync). @@ -599,27 +510,6 @@ class ThrottlingManager: self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict( "THROTTLING_SCOPES" ) - # Cheap pre-filter for the backoff methods: the union of the global - # triggers and every per-scope override. A response whose status (or an - # exception type) is not in the union cannot trigger backoff for any - # scope, so its scopes need not be resolved (see get_response_backoff - # and get_exception_backoff). Each scope still makes the final decision - # via its own triggers_backoff_* methods. - self._any_backoff_http_codes: set[int] = set(self._backoff_http_codes) - self._any_backoff_exceptions: tuple[type[BaseException], ...] = ( - self._backoff_exceptions - ) - for scope_config in self._scopes_config.values(): - backoff = scope_config.get("backoff") or {} - if "http_codes" in backoff: - self._any_backoff_http_codes.update( - int(code) for code in backoff["http_codes"] - ) - if "exceptions" in backoff: - self._any_backoff_exceptions += tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in backoff["exceptions"] - ) # Ordered by least-recently-used first (see get_scope_manager), so the # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT). self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = ( @@ -675,88 +565,28 @@ class ThrottlingManager: scope_ids = sorted(iter_scopes(scopes)) return "+".join(scope_ids) if scope_ids else "" - def _cached_scope_values( - self, request: Request - ) -> list[tuple[ScopeID, float | None]]: - """Return the ``(scope_id, quota_amount)`` pairs of *request*, reading - the scopes persisted on ``request.meta`` by :meth:`get_scopes` (via - :func:`scope_cache`) and falling back to :meth:`_resolve_scopes_sync`. + def get_resolved_scopes(self, request: Request) -> RequestScopes: + """Return the scopes under which *request* was (or will be) sent, + reusing those persisted on ``request.meta`` by an earlier + :meth:`get_scopes` call (see :func:`scope_cache`) and falling back to + :meth:`_resolve_scopes_sync` only when none were persisted. - The persisted value is authoritative here: a request reaching the - readiness API has already been enqueued and filed under the queue for - these very scopes. - """ - if _RESOLVED_SCOPES_META_KEY in request.meta: - scopes = cast("RequestScopes", request.meta[_RESOLVED_SCOPES_META_KEY]) - else: - scopes = self._resolve_scopes_sync(request) - return list(iter_scope_values(scopes)) - - async def _scopes(self, request: Request) -> RequestScopes: - """Return the scopes of *request*, reusing those persisted on - ``request.meta`` by an earlier :meth:`get_scopes` call (see - :func:`scope_cache`) and resolving them only as a fallback. - - Used by the backoff methods so that a request whose scopes were already - resolved (at enqueue time, or by :meth:`acquire`) is not resolved again - — which, for a request restored from a disk queue, would also risk - attributing the backoff to different scopes than the ones it was sent - under. + The persisted value is authoritative: a request reaching a reader of + this method (the readiness API, or a component reacting to its response) + has already been enqueued and, for a request restored from a disk queue, + re-resolving could attribute it to different scopes than the ones it was + sent under. """ if _RESOLVED_SCOPES_META_KEY in request.meta: return cast("RequestScopes", request.meta[_RESOLVED_SCOPES_META_KEY]) - return await self.get_scopes(request) + return self._resolve_scopes_sync(request) - async def get_initial_backoff(self) -> BackoffData: - return None - - async def get_response_backoff(self, response: Response) -> BackoffData: - assert response.request is not None - if response.request.meta.get("throttling_dont_track"): - return None - if response.status not in self._any_backoff_http_codes: - return None - scopes = await self._scopes(response.request) - matched = [ - scope - for scope in iter_scopes(scopes) - if self.get_scope_manager(scope).triggers_backoff_for_status( - response.status - ) - ] - if not matched: - return None - if delay := self.get_response_delay(response): - return {scope: {"delay": delay} for scope in matched} - return matched - - def get_response_delay(self, response: Response) -> float | None: - """Return the throttling delay requested by the response.""" - retry_after = _parse_retry_after(response) - ratelimit_reset = _parse_ratelimit_reset(response) - if retry_after is None and ratelimit_reset is None: - return None - if retry_after is not None and ratelimit_reset is not None: - return max(retry_after, ratelimit_reset) - if retry_after is not None: - return retry_after - assert ratelimit_reset is not None - return ratelimit_reset - - async def get_exception_backoff( - self, request: Request, exception: Exception - ) -> BackoffData: - if request.meta.get("throttling_dont_track"): - return None - if not isinstance(exception, self._any_backoff_exceptions): - return None - scopes = await self._scopes(request) - matched = [ - scope - for scope in iter_scopes(scopes) - if self.get_scope_manager(scope).triggers_backoff_for_exception(exception) - ] - return matched or None + def _cached_scope_values( + self, request: Request + ) -> list[tuple[ScopeID, float | None]]: + """Return the ``(scope_id, quota_amount)`` pairs of *request*, from the + scopes returned by :meth:`get_resolved_scopes`.""" + return list(iter_scope_values(self.get_resolved_scopes(request))) # -- Scope-state coordination (called from the request lifecycle) -------- @@ -948,42 +778,29 @@ class ThrottlingManager: logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") return deadline - async def process_response(self, response: Response) -> None: - data = await self.get_response_backoff(response) - self._apply_backoff(data) - - async def process_exception(self, request: Request, exception: Exception) -> None: - data = await self.get_exception_backoff(request, exception) - self._apply_backoff(data) - - def _apply_backoff(self, data: BackoffData) -> None: - if data is None: - return - if isinstance(data, dict): - items: Iterable[tuple[ScopeID, Any]] = data.items() - else: - items = ((scope_id, None) for scope_id in iter_scopes(data)) - for scope_id, entry in items: - manager = self.get_scope_manager(scope_id) - delay = consumed = remaining = None - if isinstance(entry, dict): - delay = entry.get("delay") - consumed = entry.get("consumed") - remaining = entry.get("remaining") - # A dict entry that only reports quota usage reconciles the quota - # without counting as a backoff trigger. - quota_only = ( - isinstance(entry, dict) - and delay is None - and (consumed is not None or remaining is not None) - ) - if consumed is not None or remaining is not None: - manager.reconcile_quota(consumed=consumed, remaining=remaining) - if quota_only: - continue + def back_off( + self, + scopes: RequestScopes, + *, + delay: float | None = None, + cap: bool = True, + ) -> None: + for scope_id in iter_scopes(scopes): if self._debug: logger.debug(f"Backoff for scope {scope_id} (delay: {delay})") - manager.record_backoff(delay=delay) + self.get_scope_manager(scope_id).record_backoff(delay=delay, cap=cap) + + def reconcile_quota( + self, + scopes: RequestScopes, + *, + consumed: float | None = None, + remaining: float | None = None, + ) -> None: + for scope_id in iter_scopes(scopes): + self.get_scope_manager(scope_id).reconcile_quota( + consumed=consumed, remaining=remaining + ) def _on_robots_parsed(self, robotparser: Any, request: Request) -> None: """Honor a robots.txt ``Crawl-delay`` on the :signal:`robots_parsed` @@ -1049,11 +866,9 @@ class ThrottlingManager: manager.set_concurrency(1) def delay_scope(self, scope_id: ScopeID, delay: float) -> None: - if self._debug: - logger.debug(f"Delaying scope {scope_id} for {delay:.2f}s") # Like a Retry-After / RateLimit-Reset header, this is a hard minimum # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. - self.get_scope_manager(scope_id).record_backoff(delay=float(delay), cap=False) + self.back_off(scope_id, delay=float(delay), cap=False) def get_scope_delay(self, scope_id: ScopeID) -> float: return self.get_scope_manager(scope_id).get_base_delay() From 7ae19cc2a2ad24cbd2764700122ed333416835b9 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 12:26:53 +0200 Subject: [PATCH 30/55] WIP --- docs/topics/components.rst | 1 + docs/topics/settings.rst | 2 + docs/topics/throttling.rst | 41 ++++++----- scrapy/core/engine.py | 13 +++- scrapy/downloadermiddlewares/redirect.py | 23 +++++++ scrapy/pqueues.py | 8 ++- scrapy/settings/default_settings.py | 2 + scrapy/throttling.py | 86 +++++++++++++----------- 8 files changed, 116 insertions(+), 60 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 025956e5e..d122271fa 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -26,6 +26,7 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_START_MEMORY_QUEUE` - :setting:`SPIDER_MIDDLEWARES` - :setting:`THROTTLING_MANAGER` +- :setting:`THROTTLING_SCOPE_MANAGER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d6b9c2623..964bd3ade 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -861,6 +861,7 @@ Default: "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, + "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 630, "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, @@ -1255,6 +1256,7 @@ Default: "scrapy.extensions.feedexport.FeedExporter": 0, "scrapy.extensions.logstats.LogStats": 0, "scrapy.extensions.spiderstate.SpiderState": 0, + "scrapy.extensions.throttle.AutoThrottle": 0, } A dict containing the extensions available by default in Scrapy, and their diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 09b8e4113..2401c5321 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -140,12 +140,7 @@ A **backoff trigger** is a response whose status code is in :setting:`BACKOFF_HTTP_CODES` or a download exception whose type is in :setting:`BACKOFF_EXCEPTIONS`. On each trigger: -#. If the response carries a :ref:`Retry-After or RateLimit-Reset - ` value, that value (capped at - :setting:`BACKOFF_MAX_DELAY`) is honored as a hard minimum: no request is - sent for the scope until it elapses. - -#. Otherwise the delay grows exponentially: +#. The delay grows exponentially: .. code-block:: text @@ -154,6 +149,13 @@ A **backoff trigger** is a response whose status code is in :setting:`BACKOFF_JITTER` is then applied so that requests that backed off together do not retry in lockstep. +#. If the response carries a :ref:`Retry-After or RateLimit-Reset + ` value, the scope is *also* held back until that + time (capped at :setting:`BACKOFF_MAX_DELAY`) before its next request. This + is a one-time gate, on top of the exponential step above: it honors the + header for the next request without turning a short header value into a + long-standing delay for every later request. + **Recovery** is linear: after a scope goes a full :setting:`BACKOFF_WINDOW` without a new trigger, its delay drops by one :setting:`BACKOFF_DELAY_FACTOR` step toward the configured value, and keeps dropping one step per quiet window @@ -227,17 +229,16 @@ setting: Target number of backoff responses per :setting:`BACKOFF_WINDOW` that :ref:`rampup ` aims for when probing the rate limit of a scope. - Can be a range like ``[1, 3]``. For every :setting:`BACKOFF_WINDOW` that stays **below** :setting:`RAMPUP_BACKOFF_TARGET` backoff triggers, rampup increases throughput one step: it first lowers the delay, and once the delay reaches its minimum it raises the concurrency limit above the ``"min_concurrency"`` floor of the -scope. Windows that hit the target hold the current rate, and windows that -exceed it let normal :ref:`backoff ` reduce the rate. The result is a -rate that converges on roughly :setting:`RAMPUP_BACKOFF_TARGET` rate-limit -responses per window — the most throughput a scope allows without being -penalized. +scope. Windows that reach or exceed the target do not ramp up; the rate is +reduced by normal :ref:`backoff ` (which grows the delay) instead. +Rampup only ever probes upward, so the rate settles around the most throughput +a scope allows while triggering fewer than :setting:`RAMPUP_BACKOFF_TARGET` +rate-limit responses per window. Rampup behavior can be fine-tuned per scope by giving ``"rampup"`` a dict instead of ``True``: @@ -248,7 +249,7 @@ instead of ``True``: THROTTLING_SCOPES = { "api.toscrape.com": { "rampup": { - "backoff_target": [1, 3], # overrides RAMPUP_BACKOFF_TARGET + "backoff_target": 1, # overrides RAMPUP_BACKOFF_TARGET "delay_factor": 0.5, # multiply the delay by this on each ramp-up step "min_delay": 0.05, # do not ramp the delay below this }, @@ -267,8 +268,9 @@ Servers may include `Retry-After or `RateLimit-Reset `__ headers to indicate when you should make your next request. These headers are -respected automatically during :ref:`backoff `, using their values as -minimum delays (capped at :setting:`BACKOFF_MAX_DELAY`). +respected automatically during :ref:`backoff `: the scope's next +request is held back until the indicated time (capped at +:setting:`BACKOFF_MAX_DELAY`), on top of the usual exponential backoff step. .. seealso:: :setting:`REDIRECT_MAX_DELAY` @@ -304,12 +306,17 @@ You can delay a :ref:`throttling scope ` on demand through :meth:`crawler.throttler.delay_scope() `: +.. skip: next + .. code-block:: python crawler.throttler.delay_scope("example.com", 30.0) -This holds back every request of the scope for at least the given number of -seconds, counted as a :ref:`backoff ` trigger. +This holds back the scope's next request for at least the given number of +seconds and registers a :ref:`backoff ` trigger. Like a +``Retry-After`` header, it is a one-time delay rather than a permanent one (the +scope's delay also grows by one backoff step and then recovers); call it again, +e.g. on each matching response, to keep a scope slowed down for longer. It is useful to react to situations that :ref:`automatic backoff ` cannot detect on its own, such as a soft block that comes back as a ``200`` diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b18ebf0d0..e06b52417 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -283,6 +283,12 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False + # pause() cancels the coalesced throttling wakeup and the loop stops + # re-running itself while paused, so re-run it here: this re-arms the + # wakeup and keeps a crawl held only by a time-based throttling gate + # (nothing in flight to re-run the loop from _download) from stalling. + if self._slot is not None: + self._slot.nextcall.schedule() async def _process_start_next(self) -> None: """Processes the next item or request from Spider.start(). @@ -553,8 +559,13 @@ class ExecutionEngine: ) async def _enqueue_request_async(self, request: Request) -> None: - assert self._slot is not None + # The counter is incremented in _schedule_request before this coroutine + # is scheduled, so it must be decremented here even on an early exit, + # otherwise spider_is_idle() never reports idle. The slot can be torn + # down (spider stopping) between the increment and this running. try: + if self._slot is None: + return stored = await self._slot.scheduler.enqueue_request_async(request) # type: ignore[attr-defined] if not stored: self.signals.send_catch_log( diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 45af5c67a..40fa5511c 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -7,6 +7,7 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy import signals +from scrapy.downloadermiddlewares.backoff import _parse_retry_after from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response from scrapy.spidermiddlewares.referer import RefererMiddleware @@ -36,6 +37,7 @@ class BaseRedirectMiddleware: raise NotConfigured self.max_redirect_times: int = settings.getint("REDIRECT_MAX_TIMES") + self.max_delay: float = settings.getfloat("REDIRECT_MAX_DELAY") self.priority_adjust: int = settings.getint("REDIRECT_PRIORITY_ADJUST") self._referer_spider_middleware: RefererMiddleware | None = None @@ -174,9 +176,30 @@ class BaseRedirectMiddleware: del redirect_request.headers["Authorization"] self.handle_referer(redirect_request, response) + self._apply_retry_after(redirect_request, response) return redirect_request + def _apply_retry_after(self, redirect_request: Request, response: Response) -> None: + """Delay the redirect when *response* carries a ``Retry-After`` header. + + The delay is capped at :setting:`REDIRECT_MAX_DELAY` and applied through + the :reqmeta:`throttling_delay` request metadata key, which holds back + this request only (without counting as a :ref:`backoff ` + trigger for its scopes). :setting:`REDIRECT_MAX_DELAY` set to ``0`` + disables it. + """ + if not self.max_delay: + return + retry_after = _parse_retry_after(response) + if retry_after is None: + return + redirect_request.meta["throttling_delay"] = min(retry_after, self.max_delay) + # This is a fresh request that may inherit an already-honored delay from + # its source request's meta; clear that state so the new delay applies. + redirect_request.meta.pop("_throttling_delayed", None) + redirect_request.meta.pop("_throttling_delay_deadline", None) + def _redirect_request_using_get( self, request: Request, response: Response, redirect_url: str ) -> Request: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 60e799ea7..9e6c54edf 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -238,10 +238,14 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None + # Mirror pop(): at a given priority it drains the regular queue before + # the start queue, so peek() must report the regular head first for the + # two to agree on "the next request" (a throttling-aware queue relies on + # this to check readiness of the exact request pop() will return). try: - queue = self._start_queues[self.curprio] - except KeyError: queue = self.queues[self.curprio] + except KeyError: + queue = self._start_queues[self.curprio] # Protocols can't declare optional members return cast("Request", queue.peek()) # type: ignore[attr-defined] diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f6e677e8b..7df8ece5e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -179,6 +179,7 @@ __all__ = [ "RANDOMIZE_DOWNLOAD_DELAY", "REACTOR_THREADPOOL_MAXSIZE", "REDIRECT_ENABLED", + "REDIRECT_MAX_DELAY", "REDIRECT_MAX_TIMES", "REDIRECT_PRIORITY_ADJUST", "REFERER_ENABLED", @@ -531,6 +532,7 @@ RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 REDIRECT_ENABLED = True +REDIRECT_MAX_DELAY = 120.0 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 5b4079746..b885bebea 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -57,7 +57,7 @@ class RampupConfig(TypedDict, total=False): :setting:`RAMPUP_BACKOFF_TARGET`). """ - backoff_target: float | list[float] + backoff_target: float delay_factor: float min_delay: float @@ -365,24 +365,28 @@ class ThrottlingManagerProtocol(Protocol): *scopes* accepts the same shapes as the output of :meth:`get_scopes` (typically the result of :meth:`get_resolved_scopes` for a request). - When *delay* is ``None`` an exponential backoff step is applied; when - given, *delay* is a hard minimum delay in seconds (e.g. from a - :ref:`Retry-After ` header). *cap* limits *delay* to - :setting:`BACKOFF_MAX_DELAY`; set it to ``False`` for trusted, - programmatic delays. + An exponential backoff step is always applied to the scope's delay. + When *delay* is given, the scope is *additionally* held back for at + least *delay* seconds before its next request: a one-time gate (e.g. + from a :ref:`Retry-After ` header), not a change to the + steady-state delay. *cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`; + set it to ``False`` for trusted, programmatic delays. """ def delay_scope(self, scope_id: str, delay: float) -> None: - """Hold back every request of the scope identified by *scope_id* for at - least *delay* seconds, counted as a :ref:`backoff ` trigger - for the scope. + """Hold back the scope identified by *scope_id* for at least *delay* + seconds before its next request, and register a :ref:`backoff + ` trigger for the scope. + + Like a :ref:`Retry-After ` response header, this is a + one-time delay (the scope's steady-state delay grows by one backoff + step and then recovers), not a permanent one; call it again to keep a + scope slowed down for longer. This is shorthand for :meth:`back_off(scope_id, delay=delay, - cap=False) `: the programmatic equivalent of a - :ref:`Retry-After ` response header. Unlike those headers, - *delay* is **not** capped at :setting:`BACKOFF_MAX_DELAY`: that cap - guards against untrusted input, whereas a ``delay_scope`` call is - trusted. + cap=False) `. Unlike a ``Retry-After`` header, *delay* is + **not** capped at :setting:`BACKOFF_MAX_DELAY`: that cap guards against + untrusted input, whereas a ``delay_scope`` call is trusted. """ def reconcile_quota( @@ -866,8 +870,9 @@ class ThrottlingManager: manager.set_concurrency(1) def delay_scope(self, scope_id: ScopeID, delay: float) -> None: - # Like a Retry-After / RateLimit-Reset header, this is a hard minimum - # delay; unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. + # Like a Retry-After / RateLimit-Reset header, this gates the scope's + # next request by delay seconds (a one-time hold, not a steady-state + # delay); unlike those, it is trusted, so it bypasses BACKOFF_MAX_DELAY. self.back_off(scope_id, delay=float(delay), cap=False) def get_scope_delay(self, scope_id: ScopeID) -> float: @@ -1076,8 +1081,10 @@ class ThrottlingScopeManager: :setting:`BACKOFF_EXCEPTIONS` exception) the delay grows exponentially by :setting:`BACKOFF_DELAY_FACTOR`, bounded by :setting:`BACKOFF_MIN_DELAY` and :setting:`BACKOFF_MAX_DELAY`, with :setting:`BACKOFF_JITTER` applied. - A ``Retry-After`` / ``RateLimit-Reset`` delay is honored as a hard - minimum (capped at :setting:`BACKOFF_MAX_DELAY`). + A ``Retry-After`` / ``RateLimit-Reset`` delay additionally holds the + scope back until that time before its next request (a one-time gate, + capped at :setting:`BACKOFF_MAX_DELAY`), without becoming the + steady-state delay. - After :setting:`BACKOFF_WINDOW` seconds without a new trigger, the delay recovers one step at a time back towards the base delay. @@ -1146,8 +1153,10 @@ class ThrottlingScopeManager: rampup = config.get("rampup") self._rampup_enabled: bool = bool(rampup) rampup_config: dict[str, Any] = rampup if isinstance(rampup, dict) else {} - self._rampup_target: tuple[float, float] = self._parse_target( - rampup_config.get("backoff_target", settings.get("RAMPUP_BACKOFF_TARGET")) + self._rampup_target: float = float( + rampup_config.get( + "backoff_target", settings.getfloat("RAMPUP_BACKOFF_TARGET") + ) ) self._rampup_delay_factor: float = float(rampup_config.get("delay_factor", 0.5)) self._rampup_min_delay: float = float(rampup_config.get("min_delay", 0.0)) @@ -1191,12 +1200,6 @@ class ThrottlingScopeManager: def _now(now: float | None) -> float: return time.monotonic() if now is None else now - @staticmethod - def _parse_target(value: Any) -> tuple[float, float]: - if isinstance(value, (list, tuple)): - return float(value[0]), float(value[1]) - return float(value), float(value) - @staticmethod def _apply_jitter(value: float, jitter: float | list[float]) -> float: if isinstance(jitter, (list, tuple)): @@ -1252,7 +1255,7 @@ class ThrottlingScopeManager: # delay or jump the concurrency limit by several steps at once). elapsed_windows = int((now - self._rampup_window_start) // self._window) self._rampup_window_start += elapsed_windows * self._window - if self._rampup_backoffs < self._rampup_target[0]: + if self._rampup_backoffs < self._rampup_target: self._rampup_step() self._rampup_backoffs = 0 @@ -1261,10 +1264,12 @@ class ThrottlingScopeManager: if self._backoff_level > 0: return if self._delay > self._rampup_min_delay: + # Lower only the effective delay while probing for headroom; the + # configured base delay is left untouched so it stays the recovery + # target on backoff and the value reported by get_base_delay(). self._delay = max( self._rampup_min_delay, self._delay * self._rampup_delay_factor ) - self._base_delay = min(self._base_delay, self._delay) else: # Rampup is only enabled with a concurrency limit set. assert self._concurrency is not None @@ -1368,20 +1373,21 @@ class ThrottlingScopeManager: self._backoff_level += 1 self._rampup_backoffs += 1 if delay is not None: + # A hard delay (e.g. a Retry-After header) is a one-time gate: hold + # the scope back for at least this long *once*, matching the HTTP + # semantics of "do not retry before this time". It is deliberately + # not turned into the steady-state inter-request delay; the delay + # still grows by one exponential step below (and recovers over + # BACKOFF_WINDOW), so a small Retry-After does not become a long + # standing delay for every later request. hard = min(float(delay), self._max_delay) if cap else float(delay) self._in_backoff_until = now + hard - self._delay = max(self._delay, hard, self._min_delay) - if cap: - self._delay = min(self._delay, self._max_delay) - else: - grown = ( - self._delay * self._delay_factor if self._delay > 0 else self._min_delay - ) - grown = max(self._min_delay, grown) - grown = min(grown, self._max_delay) - self._delay = min( - self._apply_jitter(grown, self._backoff_jitter), self._max_delay - ) + grown = self._delay * self._delay_factor if self._delay > 0 else self._min_delay + grown = max(self._min_delay, grown) + grown = min(grown, self._max_delay) + self._delay = min( + self._apply_jitter(grown, self._backoff_jitter), self._max_delay + ) self._next_allowed_time = now + self._delay def reconcile_quota( From fefa6b651e74bcfe612322ce263f42120c91fdca Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 12:43:16 +0200 Subject: [PATCH 31/55] WIP --- scrapy/core/engine.py | 5 ++++- scrapy/pqueues.py | 2 ++ scrapy/throttling.py | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index e06b52417..2469866b9 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -633,10 +633,13 @@ class ExecutionEngine: assert self._slot is not None # typing assert self.spider is not None - self._slot.add_request(request) throttler = self.crawler.throttler assert throttler is not None + # A throttling-aware scheduler reserves the request (a concurrency slot) + # before handing it here, so everything past this point runs inside the + # try/finally that releases it, even if add_request() were to raise. try: + self._slot.add_request(request) yield self._acquire_throttling(request) result: Response | Request if self._downloader_fetch_needs_spider: diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 9e6c54edf..8c11db2f4 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -603,6 +603,8 @@ class ThrottlingAwarePriorityQueue: exc_info=True, extra={"spider": getattr(self.crawler, "spider", None)}, ) + if self.crawler.stats is not None: + self.crawler.stats.inc_value("scheduler/unserializable") def _select( self, diff --git a/scrapy/throttling.py b/scrapy/throttling.py index b885bebea..ff1398c00 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -511,8 +511,8 @@ class ThrottlingManager: self._default_scope_manager_cls = load_object( crawler.settings["THROTTLING_SCOPE_MANAGER"] ) - self._scopes_config: dict[str, dict[str, Any]] = crawler.settings.getdict( - "THROTTLING_SCOPES" + self._scopes_config: dict[str, dict[str, Any]] = self._merge_download_slots( + crawler.settings ) # Ordered by least-recently-used first (see get_scope_manager), so the # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT). @@ -527,6 +527,35 @@ class ThrottlingManager: Request, list[tuple[ThrottlingScopeManagerProtocol, float | None]] ] = WeakKeyDictionary() + @staticmethod + def _merge_download_slots(settings: BaseSettings) -> dict[str, dict[str, Any]]: + """Return the effective per-scope configuration, merging the deprecated + :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLING_SCOPES`. + + Each ``DOWNLOAD_SLOTS`` entry is translated to a throttling scope keyed + by the same slot name (the default manager keys domain scopes by + ``netloc``, which is what download slots used too): ``concurrency`` and + ``delay`` map directly, and the ``randomize_delay`` boolean maps to a + ``jitter`` magnitude (the historical ±50%, or none). An explicit + ``THROTTLING_SCOPES`` entry for the same scope takes precedence over the + translated one. The deprecation warning is emitted by the downloader. + """ + scopes: dict[str, dict[str, Any]] = { + scope_id: dict(config) + for scope_id, config in settings.getdict("THROTTLING_SCOPES").items() + } + for slot_id, slot_config in settings.getdict("DOWNLOAD_SLOTS").items(): + translated: dict[str, Any] = {} + if "concurrency" in slot_config: + translated["concurrency"] = slot_config["concurrency"] + if "delay" in slot_config: + translated["delay"] = slot_config["delay"] + if "randomize_delay" in slot_config: + translated["jitter"] = 0.5 if slot_config["randomize_delay"] else 0.0 + # An explicit THROTTLING_SCOPES entry wins over the translated one. + scopes[slot_id] = {**translated, **scopes.get(slot_id, {})} + return scopes + @scope_cache async def get_scopes(self, request: Request) -> RequestScopes: return self._resolve_scopes_sync(request) From 61c9cc33ffcd2183c1e6be41629a8c428f8d15e9 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 13:23:00 +0200 Subject: [PATCH 32/55] WIP --- docs/topics/components.rst | 2 +- docs/topics/downloader-middleware.rst | 17 +-------- docs/topics/request-response.rst | 2 +- docs/topics/settings.rst | 3 +- docs/topics/throttling.rst | 49 ++----------------------- scrapy/downloadermiddlewares/backoff.py | 8 ++-- scrapy/settings/default_settings.py | 2 +- tests/test_throttling.py | 4 +- 8 files changed, 13 insertions(+), 74 deletions(-) diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d122271fa..d7aa5642a 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -10,7 +10,6 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: - :setting:`ADDONS` -- :setting:`TWISTED_DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` - :setting:`DOWNLOADER_MIDDLEWARES` - :setting:`DUPEFILTER_CLASS` @@ -27,6 +26,7 @@ That includes the classes that you may assign to the following settings: - :setting:`SPIDER_MIDDLEWARES` - :setting:`THROTTLING_MANAGER` - :setting:`THROTTLING_SCOPE_MANAGER` +- :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 83f28cf2a..fa353d126 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -174,22 +174,7 @@ For a list of the components enabled by default (and their orders) see the BackoffMiddleware ----------------- -.. module:: scrapy.downloadermiddlewares.backoff - :synopsis: Backoff Downloader Middleware - -.. class:: BackoffMiddleware - - This middleware feeds download outcomes into :ref:`throttling `: - for every response matching :setting:`BACKOFF_HTTP_CODES` and every download - exception matching :setting:`BACKOFF_EXCEPTIONS`, it makes the request's - :ref:`throttling scopes ` :ref:`back off `. - - It runs below - :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` so that it - observes rate-limiting responses (``429``, ``503``, …) before they are - turned into retries. - - See :ref:`throttling` for details. +.. autoclass:: scrapy.downloadermiddlewares.backoff.BackoffMiddleware .. _cookies-mw: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 0d8441900..f4d2d0eaa 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -713,6 +713,7 @@ Those are: * :reqmeta:`dont_obey_robotstxt` * :reqmeta:`dont_redirect` * :reqmeta:`dont_retry` +* :reqmeta:`dont_throttle` * :reqmeta:`download_fail_on_dataloss` * :reqmeta:`download_latency` * :reqmeta:`download_maxsize` @@ -733,7 +734,6 @@ Those are: * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` * :reqmeta:`throttling_delay` -* :reqmeta:`throttling_dont_track` * :reqmeta:`throttling_scopes` * :reqmeta:`verbatim_url` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 964bd3ade..6d057626a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -861,7 +861,7 @@ Default: "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, - "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 630, + "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 650, "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, @@ -2039,7 +2039,6 @@ See :ref:`asyncio-without-reactor` for more information about this mode. .. versionadded:: 2.15.0 - .. setting:: TWISTED_REACTOR TWISTED_REACTOR diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 2401c5321..50f6ca5a5 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -61,7 +61,6 @@ When configuring these settings, note that: more complex domain grouping strategies, see :ref:`alternative-domain-throttling`. - .. setting:: THROTTLING_SCOPES .. _per-domain-throttling: @@ -89,7 +88,6 @@ the :ref:`tutorial `, so that they are crawled faster while the Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are covered later on. - .. _backoff: Backoff @@ -417,7 +415,7 @@ Without a higher priority, a backlog of requests ahead of it in a FIFO queue could keep it waiting well past the configured delay; a higher priority puts it at the front of the queue, so it goes out right after its delay. -.. reqmeta:: throttling_dont_track +.. reqmeta:: dont_throttle Excluding a request from throttling state ----------------------------------------- @@ -425,12 +423,12 @@ Excluding a request from throttling state Some requests (authentication flows, one-off API calls, file downloads) should not influence throttling state even if they get a :setting:`BACKOFF_HTTP_CODES` response or raise a :setting:`BACKOFF_EXCEPTIONS` exception. Set the -``throttling_dont_track`` request metadata key to ``True`` to process such a +:reqmeta:`dont_throttle` request metadata key to ``True`` to process such a request normally without letting its outcome trigger :ref:`backoff `: .. code-block:: python - Request("https://example.com/login", meta={"throttling_dont_track": True}) + Request("https://example.com/login", meta={"dont_throttle": True}) .. _throttling-scopes: @@ -1080,47 +1078,6 @@ Additional settings long-running crawls. Set to ``0`` to never evict. Scopes in active backoff are never evicted. -.. _autothrottle-migration: - -Migrating from AutoThrottle -=========================== - -The ``AutoThrottle`` extension is deprecated in favor of the throttling and -:ref:`backoff ` system described here, which is always active and does -not need to be enabled. - -Setting ``AUTOTHROTTLE_ENABLED`` to ``True`` still works but logs a -deprecation warning. To migrate, drop the ``AUTOTHROTTLE_*`` settings and use -the following equivalents: - -.. list-table:: - :header-rows: 1 - - * - AutoThrottle - - Throttling - * - ``AUTOTHROTTLE_ENABLED = True`` - - No equivalent; throttling is always active. - * - ``AUTOTHROTTLE_START_DELAY`` - - :setting:`DOWNLOAD_DELAY` - * - ``AUTOTHROTTLE_MAX_DELAY`` - - :setting:`BACKOFF_MAX_DELAY` - * - ``AUTOTHROTTLE_TARGET_CONCURRENCY`` - - :ref:`rampup ` (``"rampup": True``) - * - ``AUTOTHROTTLE_DEBUG = True`` - - :setting:`THROTTLING_DEBUG` ``= True`` - * - ``autothrottle_dont_adjust_delay`` (request meta) - - :reqmeta:`throttling_dont_track` (request meta) - -AutoThrottle adjusted the delay of each download slot based on response -latency. The new system does not measure latency; instead, it reacts to -explicit rate-limit signals (:setting:`BACKOFF_HTTP_CODES`, -:setting:`BACKOFF_EXCEPTIONS`, :ref:`Retry-After / RateLimit-Reset -`) and, with :ref:`rampup `, probes for the -fastest rate a scope tolerates. If you specifically need latency-based control, -implement a custom :ref:`throttling scope manager -`. - - .. _throttling-api: API diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 173ef1e62..da6f2a9f1 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -69,9 +69,7 @@ class BackoffMiddleware: ` to back off the request's scopes through its :meth:`~scrapy.throttling.ThrottlingManagerProtocol.back_off` API. - It sits below :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` - in :setting:`DOWNLOADER_MIDDLEWARES_BASE` so it sees rate-limiting responses - (429, 503, …) before the retry middleware turns them into new requests. + See :ref:`throttling` for details. """ def __init__(self, crawler: Crawler): @@ -118,7 +116,7 @@ class BackoffMiddleware: if ( response.status not in self._http_codes or "cached" in response.flags - or request.meta.get("throttling_dont_track") + or request.meta.get("dont_throttle") ): return response matched = [ @@ -139,7 +137,7 @@ class BackoffMiddleware: exception: Exception, spider: scrapy.Spider | None = None, ) -> None: - if request.meta.get("throttling_dont_track") or not isinstance( + if request.meta.get("dont_throttle") or not isinstance( exception, self._exceptions ): return diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7df8ece5e..a363f2ec0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -360,7 +360,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, - "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 630, + "scrapy.downloadermiddlewares.backoff.BackoffMiddleware": 650, "scrapy.downloadermiddlewares.cookies.CookiesMiddleware": 700, "scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware": 750, "scrapy.downloadermiddlewares.stats.DownloaderStats": 850, diff --git a/tests/test_throttling.py b/tests/test_throttling.py index ddc14ff9b..72cabaa0c 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -275,7 +275,7 @@ class TestThrottlingManager: @coroutine_test async def test_response_backoff_dont_track(self): manager = _manager() - response = _response(status=429, meta={"throttling_dont_track": True}) + response = _response(status=429, meta={"dont_throttle": True}) assert await manager.get_response_backoff(response) is None @pytest.mark.parametrize( @@ -295,7 +295,7 @@ class TestThrottlingManager: @coroutine_test async def test_exception_backoff_dont_track(self): manager = _manager() - request = Request("http://example.com", meta={"throttling_dont_track": True}) + request = Request("http://example.com", meta={"dont_throttle": True}) assert ( await manager.get_exception_backoff(request, DownloadTimeoutError()) is None ) From c21f8dcfc54e82d3ed9c3b2d22ea324b8239e378 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Wed, 1 Jul 2026 14:51:10 +0200 Subject: [PATCH 33/55] WIP --- docs/topics/throttling.rst | 8 ++---- scrapy/throttling.py | 52 ++++++++++++++++++++------------------ scrapy/utils/asyncio.py | 40 +++++++++++++++++++++-------- 3 files changed, 58 insertions(+), 42 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 50f6ca5a5..99a5fe0d7 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -231,8 +231,8 @@ setting: For every :setting:`BACKOFF_WINDOW` that stays **below** :setting:`RAMPUP_BACKOFF_TARGET` backoff triggers, rampup increases throughput one step: it first lowers the delay, and once the delay reaches its minimum it -raises the concurrency limit above the ``"min_concurrency"`` floor of the -scope. Windows that reach or exceed the target do not ramp up; the rate is +raises the concurrency limit of the scope. Windows that reach or exceed the +target do not ramp up; the rate is reduced by normal :ref:`backoff ` (which grows the delay) instead. Rampup only ever probes upward, so the rate settles around the most throughput a scope allows while triggering fewer than :setting:`RAMPUP_BACKOFF_TARGET` @@ -460,10 +460,6 @@ following keys: for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and :setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope. -``min_concurrency`` (:class:`int`) - Concurrency floor that :ref:`backoff ` and :ref:`rampup ` - never drop below. Defaults to ``1``. - ``delay`` (:class:`float`) Minimum seconds between requests for the scope. Defaults to :setting:`DOWNLOAD_DELAY`. diff --git a/scrapy/throttling.py b/scrapy/throttling.py index ff1398c00..eec7c9ddd 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -71,9 +71,6 @@ class ThrottlingScopeConfig(TypedDict, total=False): concurrency: int - min_concurrency: int - """Floor. Never drop below this during backoff/rampup.""" - delay: float jitter: float | list[float] @@ -875,14 +872,6 @@ class ThrottlingManager: conflicts.append(f"delay={config['delay']!r} < Crawl-delay {capped}") if config.get("concurrency") is not None and int(config["concurrency"]) > 1: conflicts.append(f"concurrency={config['concurrency']!r} > 1") - # A min_concurrency floor above 1 keeps the scope above the single-slot - # concurrency that a Crawl-delay implies, since set_concurrency(1) is - # clamped back up to it. - if ( - config.get("min_concurrency") is not None - and int(config["min_concurrency"]) > 1 - ): - conflicts.append(f"min_concurrency={config['min_concurrency']!r} > 1") if conflicts: logger.warning( f"Throttling scope {scope_id!r} is configured with {' and '.join(conflicts)}, " @@ -1120,7 +1109,7 @@ class ThrottlingScopeManager: - When the scope is configured with a ``"concurrency"`` limit (or with ``"rampup"``), no more than that many requests are allowed in flight at - once, never dropping below the ``"min_concurrency"`` floor. + once. - When the scope sets ``"rampup": True``, throughput is increased every :setting:`BACKOFF_WINDOW` that stays under :setting:`RAMPUP_BACKOFF_TARGET` @@ -1176,7 +1165,6 @@ class ThrottlingScopeManager: for exc in backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) ) self._window: float = settings.getfloat("BACKOFF_WINDOW") - self._min_concurrency: int = int(config.get("min_concurrency", 1)) # Rampup. rampup = config.get("rampup") @@ -1197,7 +1185,8 @@ class ThrottlingScopeManager: if configured_concurrency is not None: self._concurrency: int | None = int(configured_concurrency) elif self._rampup_enabled: - self._concurrency = self._min_concurrency + # Rampup starts conservative at a single slot and probes upward. + self._concurrency = 1 else: self._concurrency = _default_scope_concurrency(settings, self._id) or None # Used as the load denominator when the scope enforces no explicit @@ -1239,9 +1228,16 @@ class ThrottlingScopeManager: return value * random.uniform(1 - jitter, 1 + jitter) # noqa: S311 def _effective_delay(self) -> float: - if self._backoff_level == 0 and self._delay > 0: - return self._apply_jitter(self._delay, self._jitter) - return self._delay + # ``self._delay`` is the deterministic delay (the base delay, or the + # bounded exponential value while backing off); jitter is applied here, + # per use, so successive delays spread out without compounding and + # without piling probability mass on the min/max bounds (which clipping + # a jittered value would do). While backing off, the backoff jitter + # applies; otherwise the plain delay jitter does. + if self._delay <= 0: + return self._delay + jitter = self._backoff_jitter if self._backoff_level > 0 else self._jitter + return self._apply_jitter(self._delay, jitter) def _recover(self, now: float) -> None: if self._backoff_level == 0 or self._last_backoff_time is None: @@ -1321,9 +1317,13 @@ class ThrottlingScopeManager: self._consumed = 0.0 def can_send(self, now: float | None = None, amount: float | None = None) -> float: + # can_send() only refreshes passive, time-based state (backoff recovery + # and the quota window) to reflect the current time; it performs no + # active throughput probing. That way a readiness check (is_ready() / + # get_time_until_ready()) has no side effect on the send rate: rampup + # only advances on an actual send, from record_sent(). now = self._now(now) self._recover(now) - self._maybe_rampup(now) self._maybe_reset_quota(now) waits = [0.0] if self._in_backoff_until is not None: @@ -1348,6 +1348,9 @@ class ThrottlingScopeManager: self._last_seen = now if self._in_backoff_until is not None and now >= self._in_backoff_until: self._in_backoff_until = None + # An actual send is the cue to probe for more throughput (rampup), + # rather than a mere readiness check; see can_send(). + self._maybe_rampup(now) self._next_allowed_time = now + self._effective_delay() self._active += 1 if self._quota is not None and amount is not None: @@ -1412,12 +1415,11 @@ class ThrottlingScopeManager: hard = min(float(delay), self._max_delay) if cap else float(delay) self._in_backoff_until = now + hard grown = self._delay * self._delay_factor if self._delay > 0 else self._min_delay - grown = max(self._min_delay, grown) - grown = min(grown, self._max_delay) - self._delay = min( - self._apply_jitter(grown, self._backoff_jitter), self._max_delay - ) - self._next_allowed_time = now + self._delay + # Store the deterministic, bounded delay; jitter is applied per use in + # _effective_delay(), so it neither compounds across successive backoff + # steps nor concentrates on BACKOFF_MIN_DELAY / BACKOFF_MAX_DELAY. + self._delay = min(max(self._min_delay, grown), self._max_delay) + self._next_allowed_time = now + self._effective_delay() def reconcile_quota( self, @@ -1461,7 +1463,7 @@ class ThrottlingScopeManager: self._delay = delay def set_concurrency(self, concurrency: int) -> None: - self._concurrency = max(self._min_concurrency, int(concurrency)) + self._concurrency = max(1, int(concurrency)) self._fire_slot_waiters() def is_idle(self, now: float, max_idle: float) -> bool: diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 71aecde9c..de16ee194 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -8,7 +8,7 @@ import time from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Sequence from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred from twisted.internet.task import LoopingCall, deferLater from twisted.internet.threads import deferToThread @@ -347,6 +347,10 @@ async def wait_for_first( Unfired deferreds in the ``pending`` set are neither cancelled nor otherwise modified; the caller is responsible for any cleanup. + A deferred that fails counts as done and its failure is **not** re-raised + here (it stays on the deferred for the caller to inspect or handle), exactly + as a failed awaitable lands in the ``done`` set of :func:`asyncio.wait`. + Returns ``(set(), set())`` immediately when *deferreds* is empty. Works transparently in asyncio-reactor, non-asyncio-reactor, and @@ -377,17 +381,31 @@ async def wait_for_first( # circular import from scrapy.utils.defer import maybe_deferred_to_future # noqa: PLC0415 - timeout_deferred = ( - deferLater(reactor, timeout, lambda: None) if timeout is not None else None - ) - waiter = DeferredList( - [*deferreds, *([] if timeout_deferred is None else [timeout_deferred])], - fireOnOneCallback=True, - fireOnOneErrback=True, - consumeErrors=True, - ) + # Fire a single signal Deferred as soon as any input Deferred (or the + # timeout) completes, whether it succeeds or fails. Unlike a DeferredList + # with fireOnOneErrback (which would raise the first failure into this + # coroutine), _fire passes each result through untouched, so a failure just + # marks its Deferred as done and stays there for the caller — matching the + # asyncio branch and asyncio.wait(return_when=FIRST_COMPLETED). + signal: Deferred[None] = Deferred() + + def _fire(result: Any) -> Any: + if not signal.called: + signal.callback(None) + return result + + for d in deferreds: + d.addBoth(_fire) + + timeout_deferred: Deferred[Any] | None = None + if timeout is not None: + timeout_deferred = deferLater(reactor, timeout, lambda: None) + timeout_deferred.addBoth(_fire) + # Swallow the CancelledError from the finally-block cancel() below. + timeout_deferred.addErrback(lambda _: None) + try: - await maybe_deferred_to_future(waiter) + await maybe_deferred_to_future(signal) finally: if timeout_deferred is not None and not timeout_deferred.called: timeout_deferred.cancel() From e203c326c273dad0a306c973130f710aa4d719b6 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 06:48:46 +0200 Subject: [PATCH 34/55] WIP --- docs/topics/throttling.rst | 9 +++++++++ scrapy/throttling.py | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 99a5fe0d7..2de8cea8d 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -1044,6 +1044,14 @@ Additional settings ` that are neither domains nor IPs, e.g. custom scopes added by a :ref:`custom throttling manager `. + A scope counts as a domain only if it looks like a hostname with at least + one dot, so single-label hosts such as ``localhost`` (or intranet host + names) fall in this category and default to + :setting:`THROTTLING_SCOPE_CONCURRENCY` rather than + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Raise it, or give the host an + explicit ``concurrency`` in :setting:`THROTTLING_SCOPES`, if you need more + concurrency against such hosts (e.g. a local development server). + Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set in :setting:`THROTTLING_SCOPES` overrides this. @@ -1101,3 +1109,4 @@ API .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope +.. autofunction:: scrapy.throttling.iter_scopes diff --git a/scrapy/throttling.py b/scrapy/throttling.py index eec7c9ddd..356571a05 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -98,6 +98,14 @@ RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None] def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: + """Iterate over the scope IDs of *scopes*, whatever its form. + + :class:`~ThrottlingManagerProtocol.get_scopes` (and + :meth:`~ThrottlingManagerProtocol.get_resolved_scopes`) may return a single + scope ID, an iterable of them, a ``{scope_id: quota}`` mapping, or ``None``; + this helper normalizes any of those into an iterable of scope IDs, e.g. to + react to a request's scopes in a custom middleware. + """ if scopes is None: return () if isinstance(scopes, str): From 8e488d837c3e4ae2efe7bec86292451605170df5 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 06:59:52 +0200 Subject: [PATCH 35/55] Clarify backoff per-scope overrides --- docs/topics/throttling.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 2de8cea8d..f6e4a2100 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -105,7 +105,8 @@ The key settings are: :setting:`BACKOFF_HTTP_CODES` (default: ``[429, 502, 503, 504, 520, 521, 522, 523, 524]``) - HTTP response status codes that trigger backoff. + HTTP response status codes that trigger backoff. Can be overridden per + scope with the ``http_codes`` key (see :ref:`per-scope-backoff`). - .. setting:: BACKOFF_DELAY_FACTOR @@ -136,7 +137,10 @@ starts at its configured ``"delay"`` (:setting:`DOWNLOAD_DELAY` by default). A **backoff trigger** is a response whose status code is in :setting:`BACKOFF_HTTP_CODES` or a download exception whose type is in -:setting:`BACKOFF_EXCEPTIONS`. On each trigger: +:setting:`BACKOFF_EXCEPTIONS`. Both can be :ref:`overridden per scope +` (through the ``http_codes`` and ``exceptions`` keys), so +the same response or exception may trigger backoff for one scope and not for +another. On each trigger: #. The delay grows exponentially: @@ -198,7 +202,13 @@ The global ``BACKOFF_*`` settings can be overridden per scope with the }, } -Any key left out falls back to the matching global ``BACKOFF_*`` setting. +Every key overrides the matching global ``BACKOFF_*`` setting for that scope +(``http_codes`` overrides :setting:`BACKOFF_HTTP_CODES`, ``exceptions`` +overrides :setting:`BACKOFF_EXCEPTIONS`, ``delay_factor`` overrides +:setting:`BACKOFF_DELAY_FACTOR`, and so on), and any key left out falls back to +it. So a scope can, for example, treat an extra status code as a backoff +trigger, or stop treating one of the defaults as a trigger, independently of +every other scope. .. _rampup: @@ -967,7 +977,9 @@ Additional settings - :exc:`~scrapy.exceptions.ResponseDataLossError` List of exception classes that trigger backoff when raised while - downloading a request. Strings are interpreted as import paths. + downloading a request. Strings are interpreted as import paths. Can be + overridden per scope with the ``exceptions`` key (see + :ref:`per-scope-backoff`). .. seealso:: :setting:`RETRY_EXCEPTIONS` From 7784594f7ac84f7ffaf40b76efd27d197bf7a842 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 08:27:20 +0200 Subject: [PATCH 36/55] Rederepcate PER_IP, deprecate PER_DOMAIN --- docs/topics/broad-crawls.rst | 4 +- docs/topics/throttling.rst | 95 +++++--------- scrapy/core/downloader/__init__.py | 4 +- scrapy/core/downloader/handlers/http11.py | 10 +- scrapy/pqueues.py | 5 +- scrapy/settings/__init__.py | 9 ++ scrapy/settings/default_settings.py | 19 ++- .../templates/project/module/settings.py.tmpl | 2 +- scrapy/throttling.py | 122 +++++++++--------- scrapy/utils/test.py | 7 +- tests/test_crawler.py | 72 +++++++++++ tests/test_settings/__init__.py | 9 ++ 12 files changed, 209 insertions(+), 149 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index cace1f883..21096bdfb 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -47,8 +47,8 @@ Increase concurrency ==================== Concurrency is the number of requests that are processed in parallel. There is -a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). +a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional per-scope +(per-domain by default) limit (:setting:`THROTTLING_SCOPE_CONCURRENCY`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index f6e4a2100..accf6a590 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,15 +26,16 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN +- .. setting:: THROTTLING_SCOPE_CONCURRENCY - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``1`` - (:ref:`fallback `: ``8``)) + :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) - Maximum number of simultaneous requests per domain. + Default maximum number of simultaneous requests per :ref:`throttling scope + `. Requests are grouped by domain by default, so this is + the maximum number of simultaneous requests per domain. - It defines a number of “slots” per domain. Each slot can send 1 request at - a time: it sends a request, waits for the response, then sends the next + It defines a number of “slots” per scope. Each slot can send 1 request at a + time: it sends a request, waits for the response, then sends the next request, and so on. - .. setting:: DOWNLOAD_DELAY @@ -50,7 +51,7 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, @@ -296,7 +297,7 @@ to wait between requests. If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`. +:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency is set to ``1`` and the delay is raised to at least the ``Crawl-Delay`` value (a larger configured delay is kept), capped at :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). @@ -465,10 +466,8 @@ Its keys are scope names and its values are following keys: ``concurrency`` (:class:`int`) - Maximum number of concurrent requests for the scope. When unset, the - default depends on the kind of scope: :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` - for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and - :setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope. + Maximum number of concurrent requests for the scope. Defaults to + :setting:`THROTTLING_SCOPE_CONCURRENCY`. ``delay`` (:class:`float`) Minimum seconds between requests for the scope. Defaults to @@ -921,46 +920,30 @@ window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas Per-IP concurrency limiting --------------------------- -.. setting:: CONCURRENT_REQUESTS_PER_IP +A concurrency limit keyed by IP is just a throttling scope whose id is the +request's IP, with a ``concurrency`` limit. A request then carries two scopes, +its domain and its IP, and is only sent when **both** allow it (see +:ref:`multiple-throttling-scopes`). -A concurrency limit keyed by IP is a throttling scope whose id is the request's -IP. A request then carries two scopes, its domain and its IP, and is only sent -when **both** allow it (see :ref:`multiple-throttling-scopes`). +- Implement a :ref:`throttling manager ` that adds + the request's IP as a second scope: -Set :setting:`CONCURRENT_REQUESTS_PER_IP` (default: ``0``, disabled) to a -positive number to enable this built in: the throttler adds the IP of each -request, resolved from the DNS cache, as a second scope, limited to that many -concurrent requests. IP scopes whose ``concurrency`` is left unset default to -:setting:`CONCURRENT_REQUESTS_PER_IP`. + .. code-block:: python -.. note:: Enabling :setting:`CONCURRENT_REQUESTS_PER_IP` requires the - :ref:`throttling-aware scheduler `. The default - :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` does not support it - and raises an error at start up if it is set, so switch - :setting:`SCHEDULER` and :setting:`SCHEDULER_PRIORITY_QUEUE` as shown - there. + import socket -For finer control (e.g. resolving IPs that are not in the DNS cache yet, or -grouping several hosts under one address), implement a :ref:`throttling manager -` that adds the request's IP as a second scope -instead: - -.. code-block:: python - - import socket - - from scrapy.throttling import ThrottlingManager, add_scope, scope_cache - from scrapy.utils.asyncio import run_in_thread - from scrapy.utils.httpobj import urlparse_cached + from scrapy.throttling import ThrottlingManager, add_scope, scope_cache + from scrapy.utils.asyncio import run_in_thread + from scrapy.utils.httpobj import urlparse_cached - class IPThrottlingManager(ThrottlingManager): - @scope_cache - async def get_scopes(self, request): - scopes = await super().get_scopes(request) - host = urlparse_cached(request).hostname - address = await run_in_thread(socket.gethostbyname, host) - return add_scope(scopes, address) + class IPThrottlingManager(ThrottlingManager): + @scope_cache + async def get_scopes(self, request): + scopes = await super().get_scopes(request) + host = urlparse_cached(request).hostname + address = await run_in_thread(socket.gethostbyname, host) + return add_scope(scopes, address) .. _throttling-settings: @@ -1048,26 +1031,6 @@ Additional settings Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. -- .. setting:: THROTTLING_SCOPE_CONCURRENCY - - :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) - - Default maximum number of concurrent requests for :ref:`throttling scopes - ` that are neither domains nor IPs, e.g. custom scopes - added by a :ref:`custom throttling manager `. - - A scope counts as a domain only if it looks like a hostname with at least - one dot, so single-label hosts such as ``localhost`` (or intranet host - names) fall in this category and default to - :setting:`THROTTLING_SCOPE_CONCURRENCY` rather than - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. Raise it, or give the host an - explicit ``concurrency`` in :setting:`THROTTLING_SCOPES`, if you need more - concurrency against such hosts (e.g. a local development server). - - Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set - in :setting:`THROTTLING_SCOPES` overrides this. - - .. setting:: THROTTLING_SCOPE_LIMIT :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6941e35cc..517be0fcd 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -251,9 +251,7 @@ class Downloader: return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - # Per-IP grouping (CONCURRENT_REQUESTS_PER_IP) is now a throttling scope - # handled by the throttler, not a downloader slot key; this fallback (used - # only when no throttler is set) keys by domain alone. + # This fallback (used only when no throttler is set) keys by domain. return urlparse_cached(request).netloc or "" async def _enqueue_request(self, request: Request) -> Response: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 30b8d73e0..49422b0bf 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -93,11 +93,10 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) # Keep enough persistent connections per host to match the highest - # per-host concurrency the throttler may admit: the per-domain and - # per-IP limits, the default "other"-scope limit, and any explicit - # THROTTLING_SCOPES concurrency. Per-scope concurrency can grow beyond - # this at runtime (rampup); the excess simply uses non-persistent - # connections. + # per-host concurrency the throttler may admit: the per-domain limit, + # the default "other"-scope limit, and any explicit THROTTLING_SCOPES + # concurrency. Per-scope concurrency can grow beyond this at runtime + # (rampup); the excess simply uses non-persistent connections. scope_concurrencies = [ scope["concurrency"] for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() @@ -106,7 +105,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): self._pool.maxPersistentPerHost = max( [ crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), - crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP"), crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), *scope_concurrencies, ] diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 8c11db2f4..75bec1a46 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -344,10 +344,7 @@ class DownloaderAwarePriorityQueue: ): if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP. ' - "Set SCHEDULER_PRIORITY_QUEUE to " - "'scrapy.pqueues.ThrottlingAwarePriorityQueue' (along with the " - "matching SCHEDULER) to use per-IP concurrency limiting." + f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' ) if slot_startprios and not isinstance(slot_startprios, dict): diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index cdf8b2f1b..932463ba9 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,6 +149,15 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ + if name == "CONCURRENT_REQUESTS_PER_IP" and ( + isinstance(self[name], int) and self[name] != 0 + ): + warnings.warn( + "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a363f2ec0..4dba40fe0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,6 +16,7 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path +from typing import Any __all__ = [ "ADDONS", @@ -51,7 +52,6 @@ __all__ = [ "CONCURRENT_ITEMS", "CONCURRENT_REQUESTS", "CONCURRENT_REQUESTS_PER_DOMAIN", - "CONCURRENT_REQUESTS_PER_IP", "COOKIES_DEBUG", "COOKIES_ENABLED", "CRAWLSPIDER_FOLLOW_LINKS", @@ -285,7 +285,6 @@ CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 -CONCURRENT_REQUESTS_PER_IP = 0 COOKIES_ENABLED = True COOKIES_DEBUG = False @@ -636,3 +635,19 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True + + +def __getattr__(name: str) -> Any: + if name == "CONCURRENT_REQUESTS_PER_IP": + import warnings # noqa: PLC0415 + + from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 + + warnings.warn( + "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return 0 + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 592fa5619..a035cc9ed 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -13,7 +13,7 @@ NEWSPIDER_MODULE = "$project_name.spiders" ROBOTSTXT_OBEY = True # Throttle crawls to be polite to websites: -CONCURRENT_REQUESTS_PER_DOMAIN = 1 +THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 # Crawl the tutorial websites faster, overriding the polite defaults above only diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 356571a05..cc38a0d47 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1,10 +1,8 @@ from __future__ import annotations import contextlib -import ipaddress import logging import random -import re import time import warnings from collections import OrderedDict @@ -18,7 +16,7 @@ from typing_extensions import Self from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.resolver import dnscache +from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object @@ -134,59 +132,67 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None -# A DNS hostname with at least one dot, e.g. "books.toscrape.com"; labels are -# alphanumeric (plus hyphens, not at the edges) and an optional trailing dot -# marks a fully-qualified name. -_HOSTNAME_RE = re.compile( - r"(?i)^(?!-)[a-z0-9-]{1,63}(? int: + """Return the default concurrency of a throttling scope that does not set + its own ``concurrency``. - -def _classify_scope(scope_id: ScopeID) -> str: - """Classify *scope_id* as ``"ip"``, ``"domain"`` or ``"other"`` to pick its - default concurrency setting (see :func:`_default_scope_concurrency`). - - A scope that parses as an IP address is ``"ip"``; one that *looks like* a - DNS hostname with at least one dot is ``"domain"``; anything else (custom - group names, single labels like ``"localhost"``) is ``"other"``. + This is :setting:`THROTTLING_SCOPE_CONCURRENCY`, except that the deprecated + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is bridged in when set at a + higher :ref:`priority `. When neither is set + explicitly, the historical :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` value is + kept for backward compatibility (its default flips to + :setting:`THROTTLING_SCOPE_CONCURRENCY` in a future version; see + :func:`_warn_on_deprecated_concurrency`). """ - host = scope_id - if host.startswith("[") and "]" in host: # bracketed IPv6, e.g. "[::1]:80" - host = host[1 : host.index("]")] - with contextlib.suppress(ValueError): - ipaddress.ip_address(host) - return "ip" - # Drop a trailing ":port" for the hostname check (an IPv4 "host:port" is - # re-tested as an IP below). - if host.count(":") == 1: - host = host.rsplit(":", 1)[0] - with contextlib.suppress(ValueError): - ipaddress.ip_address(host) - return "ip" - if "." in host and _HOSTNAME_RE.match(host): - return "domain" - return "other" - - -def _default_scope_concurrency(settings: BaseSettings, scope_id: ScopeID) -> int: - """Return the default concurrency for *scope_id* based on its - :func:`kind <_classify_scope>`. - - Domain scopes default to :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, IP - scopes to :setting:`CONCURRENT_REQUESTS_PER_IP` (falling back to the - per-domain value when per-IP limiting is off, so IP-literal crawls are not - left unbounded), and any other scope to :setting:`THROTTLING_SCOPE_CONCURRENCY`. - """ - kind = _classify_scope(scope_id) - if kind == "ip": - return settings.getint("CONCURRENT_REQUESTS_PER_IP") or settings.getint( - "CONCURRENT_REQUESTS_PER_DOMAIN" - ) - if kind == "domain": + default_priority = SETTINGS_PRIORITIES["default"] + # An unset setting has no priority (``None``); treat it as below "default" + # so it never wins over one that is at least at its default value. + domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") + domain_priority = ( + default_priority - 1 if domain_priority is None else domain_priority + ) + scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") + scope_priority = default_priority - 1 if scope_priority is None else scope_priority + if domain_priority > scope_priority: + return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + if scope_priority > domain_priority: + return settings.getint("THROTTLING_SCOPE_CONCURRENCY") + # Equal priority: on an explicit (higher-than-default) tie the new setting + # wins; when neither is set (both at "default") keep the historical + # per-domain value so existing behavior is preserved. + if domain_priority <= default_priority: return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") return settings.getint("THROTTLING_SCOPE_CONCURRENCY") +def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: + """Warn about the concurrency settings bridged by + :func:`_default_scope_concurrency`. Call once per crawl (see + :meth:`ThrottlingManager.__init__`).""" + default_priority = SETTINGS_PRIORITIES["default"] + domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") + scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") + domain_set = domain_priority is not None and domain_priority > default_priority + scope_set = scope_priority is not None and scope_priority > default_priority + if domain_set: + warnings.warn( + "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " + "THROTTLING_SCOPE_CONCURRENCY instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + elif not scope_set: + current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + future = settings.getint("THROTTLING_SCOPE_CONCURRENCY") + warnings.warn( + f"The default value of THROTTLING_SCOPE_CONCURRENCY will change " + f"from {current} to {future} in a future Scrapy version. Set " + f"THROTTLING_SCOPE_CONCURRENCY explicitly to silence this warning.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]: """Normalize *collection* (``None``, str, iterable or dict) into a dict mapping scope names to values produced by *default*.""" @@ -496,10 +502,8 @@ class ThrottlingManager: def __init__(self, crawler: Crawler) -> None: self.crawler = crawler + _warn_on_deprecated_concurrency(crawler.settings) self._debug = crawler.settings.getbool("THROTTLING_DEBUG") - # When set, each request also gets an IP scope (its resolved address), - # enforced alongside its domain scope (see _resolve_scopes_sync). - self._per_ip: int = crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") self._robotstxt_obey = crawler.settings.getbool( "ROBOTSTXT_OBEY" @@ -588,15 +592,7 @@ class ThrottlingManager: stacklevel=2, ) return cast("RequestScopes", download_slot) - netloc = urlparse_cached(request).netloc - if self._per_ip: - # Best-effort, mirroring the legacy per-IP downloader slots: use the - # cached resolved address if available, else fall back to the domain - # scope alone (the IP is not known until the host is resolved). - ip = dnscache.get(netloc) - if isinstance(ip, str) and ip != netloc: - return (netloc, ip) - return netloc + return urlparse_cached(request).netloc def get_slot_key(self, request: Request) -> str: scopes = self._resolve_scopes_sync(request) @@ -1196,7 +1192,7 @@ class ThrottlingScopeManager: # Rampup starts conservative at a single slot and probes upward. self._concurrency = 1 else: - self._concurrency = _default_scope_concurrency(settings, self._id) or None + self._concurrency = _default_scope_concurrency(settings) or None # Used as the load denominator when the scope enforces no explicit # concurrency limit (see get_load()). self._global_concurrency: int = settings.getint("CONCURRENT_REQUESTS") diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 38bd8c124..ee289b4e9 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,13 +66,16 @@ def get_crawler( will be used to populate the crawler settings with a project level priority. """ - # When needed, useful settings can be added here, e.g. ones that prevent - # deprecation warnings (see prevent_warnings). settings: dict[str, Any] = { "TELNETCONSOLE_ENABLED": False, **get_reactor_settings(), **(settings_dict or {}), } + if prevent_warnings: + # Pin the pre-deprecation default per-scope concurrency so the suite + # keeps its historical behavior and does not emit the warn-then-flip + # deprecation warning (see throttling._warn_on_deprecated_concurrency). + settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index b249c8916..d68d63392 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -5,6 +5,7 @@ import logging import re import signal import threading +import warnings from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar from unittest.mock import MagicMock @@ -52,6 +53,77 @@ def get_raw_crawler( return Crawler(spidercls or DefaultSpider, settings) +class TestScopeConcurrencyBridge: + """CONCURRENT_REQUESTS_PER_DOMAIN is deprecated and bridged into + THROTTLING_SCOPE_CONCURRENCY by setting priority (see + scrapy.throttling._default_scope_concurrency).""" + + @staticmethod + def _resolve(settings: Settings) -> int: + from scrapy.throttling import _default_scope_concurrency # noqa: PLC0415 + + return _default_scope_concurrency(settings) + + def test_neither_set_preserves_per_domain_default(self) -> None: + # Neither setting overridden: keep the historical per-domain default. + assert self._resolve(Settings()) == 8 + + def test_per_domain_overrides(self) -> None: + settings = Settings() + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") + assert self._resolve(settings) == 4 + + def test_scope_concurrency_overrides(self) -> None: + settings = Settings() + settings.set("THROTTLING_SCOPE_CONCURRENCY", 4, priority="project") + assert self._resolve(settings) == 4 + + def test_new_setting_wins_on_non_default_tie(self) -> None: + settings = Settings() + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") + settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project") + assert self._resolve(settings) == 9 + + def test_higher_priority_deprecated_setting_wins(self) -> None: + settings = Settings() + settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project") + settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="cmdline") + assert self._resolve(settings) == 4 + + def test_warns_flip_when_neither_set(self) -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="THROTTLING_SCOPE_CONCURRENCY will change", + ): + get_crawler(prevent_warnings=False) + + def test_warns_deprecated_when_per_domain_set(self) -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated", + ): + get_crawler( + settings_dict={"CONCURRENT_REQUESTS_PER_DOMAIN": 4}, + prevent_warnings=False, + ) + + def test_no_concurrency_warning_when_scope_set(self) -> None: + with warnings.catch_warnings(record=True) as recorded: + warnings.simplefilter("always") + get_crawler( + settings_dict={"THROTTLING_SCOPE_CONCURRENCY": 4}, + prevent_warnings=False, + ) + messages = [str(w.message) for w in recorded] + assert not any( + "THROTTLING_SCOPE_CONCURRENCY will change" in m for m in messages + ) + assert not any( + "CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated" in m + for m in messages + ) + + class TestBaseCrawler: @staticmethod def assertOptionIsDefault(settings: Settings, key: str) -> None: diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index ab8a20c14..3282b0591 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -769,6 +769,15 @@ def test_deprecated_dns_resolver_setting(): settings.get("DNS_RESOLVER") +def test_deprecated_concurrent_requests_per_ip_setting(): + settings = Settings({"CONCURRENT_REQUESTS_PER_IP": 1}) + with pytest.warns( + ScrapyDeprecationWarning, + match="The CONCURRENT_REQUESTS_PER_IP setting is deprecated", + ): + settings.get("CONCURRENT_REQUESTS_PER_IP") + + class Component1: pass From 8f1f66e1ebb52721996d339d0ca63bb71f5f21b5 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 09:28:01 +0200 Subject: [PATCH 37/55] Address backward compatibility issues, remove CONCURRENT_REQUESTS_PER_IP --- docs/news.rst | 4 ++-- scrapy/core/downloader/__init__.py | 6 ++++- scrapy/crawler.py | 36 +++++++++++++++++++++++++++++ scrapy/pqueues.py | 5 ---- scrapy/settings/__init__.py | 9 -------- scrapy/settings/default_settings.py | 17 -------------- 6 files changed, 43 insertions(+), 34 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b923f8135..37d0d8c8a 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -6275,7 +6275,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -8753,7 +8753,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 517be0fcd..6c5598335 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -251,7 +251,11 @@ class Downloader: return self.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - # This fallback (used only when no throttler is set) keys by domain. + # This fallback is used only when no throttler is set; it mirrors the + # historical keying (an explicit download_slot wins, else the domain). + meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) + if meta_slot is not None: + return meta_slot return urlparse_cached(request).netloc or "" async def _enqueue_request(self, request: Request) -> Response: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index cd457c22a..84f6774d5 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -105,6 +105,7 @@ class Crawler: self.addons.load_settings(self.settings) self._apply_spider_download_delay() + self._apply_spider_max_concurrent_requests() self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -189,6 +190,41 @@ class Crawler: "DOWNLOAD_DELAY", spider.download_delay, priority="spider" ) + def _apply_spider_max_concurrent_requests(self) -> None: + spider = self.spider if self.spider is not None else self.spidercls + if not hasattr(spider, "max_concurrent_requests"): + return + # Historically this attribute overrode the per-domain slot concurrency, + # which is now THROTTLING_SCOPE_CONCURRENCY (see + # scrapy.throttling._default_scope_concurrency). The old deprecation + # message pointed at CONCURRENT_REQUESTS, but that never matched its + # actual per-domain effect. + concurrency_prio = ( + self.settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") or 0 + ) + if concurrency_prio >= SETTINGS_PRIORITIES["spider"]: + warnings.warn( + "The 'max_concurrent_requests' spider attribute is deprecated. " + "It is also being ignored because THROTTLING_SCOPE_CONCURRENCY is " + "already set at spider or higher priority. Remove the " + "'max_concurrent_requests' attribute from your spider.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + else: + warnings.warn( + "The 'max_concurrent_requests' spider attribute is deprecated. Use " + "the THROTTLING_SCOPE_CONCURRENCY setting or per-domain " + "THROTTLING_SCOPES instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self.settings.set( + "THROTTLING_SCOPE_CONCURRENCY", + spider.max_concurrent_requests, + priority="spider", + ) + def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75bec1a46..a476602e9 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -342,11 +342,6 @@ class DownloaderAwarePriorityQueue: *, start_queue_cls: type[QueueProtocol] | None = None, ): - if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0: - raise ValueError( - f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP' - ) - if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError( "DownloaderAwarePriorityQueue accepts " diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 932463ba9..cdf8b2f1b 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -149,15 +149,6 @@ class BaseSettings(MutableMapping[str, Any]): :param default: the value to return if no setting is found :type default: object """ - if name == "CONCURRENT_REQUESTS_PER_IP" and ( - isinstance(self[name], int) and self[name] != 0 - ): - warnings.warn( - "The CONCURRENT_REQUESTS_PER_IP setting is deprecated, use CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - if name == "DNS_RESOLVER": warnings.warn( "The DNS_RESOLVER setting is deprecated, please use " diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 4dba40fe0..e4f9788d4 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,7 +16,6 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path -from typing import Any __all__ = [ "ADDONS", @@ -635,19 +634,3 @@ URLLENGTH_LIMIT = 2083 USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org)" WARN_ON_GENERATOR_RETURN_VALUE = True - - -def __getattr__(name: str) -> Any: - if name == "CONCURRENT_REQUESTS_PER_IP": - import warnings # noqa: PLC0415 - - from scrapy.exceptions import ScrapyDeprecationWarning # noqa: PLC0415 - - warnings.warn( - "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.", - ScrapyDeprecationWarning, - stacklevel=2, - ) - return 0 - - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From 27f239074c7c98718198f8bb38e7e66e95f00f87 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 10:05:47 +0200 Subject: [PATCH 38/55] WIP --- docs/news.rst | 8 ++-- scrapy/core/downloader/__init__.py | 65 +++++++++------------------ scrapy/extensions/throttle.py | 2 +- scrapy/pqueues.py | 14 ++---- tests/test_downloaderslotssettings.py | 2 - 5 files changed, 31 insertions(+), 60 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 37d0d8c8a..7a87e31b3 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1173,7 +1173,7 @@ Deprecations (:issue:`7005`, :issue:`7043`) - The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + ``CONCURRENT_REQUESTS_PER_DOMAIN`` instead. (:issue:`6917`, :issue:`6921`) - The ``scrapy.core.downloader.handlers.http`` module is deprecated. You @@ -1505,7 +1505,7 @@ Scrapy 2.13.3 (2025-07-02) -------------------------- - Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the + ``CONCURRENT_REQUESTS_PER_DOMAIN`` (from ``8`` to ``1``) in the default project template. (:issue:`6597`, :issue:`6918`, :issue:`6923`) @@ -3259,7 +3259,7 @@ New features ~~~~~~~~~~~~ - Settings corresponding to :setting:`DOWNLOAD_DELAY`, - :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and + ``CONCURRENT_REQUESTS_PER_DOMAIN`` and :setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`) @@ -8753,7 +8753,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` + - :setting:`CONCURRENT_REQUESTS`, ``CONCURRENT_REQUESTS_PER_DOMAIN``, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6c5598335..8e4053a92 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -26,7 +26,10 @@ if TYPE_CHECKING: from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager - from scrapy.throttling import ThrottlingScopeManagerProtocol + from scrapy.throttling import ( + ThrottlingManagerProtocol, + ThrottlingScopeManagerProtocol, + ) @dataclass(slots=True, eq=False) @@ -56,7 +59,7 @@ class _DeprecatedSlotView: self, downloader: Downloader, key: str, - scope: ThrottlingScopeManagerProtocol | None, + scope: ThrottlingScopeManagerProtocol, ) -> None: self._downloader = downloader self._key = key @@ -84,20 +87,15 @@ class _DeprecatedSlotView: @property def delay(self) -> float: - if self._scope is not None: - return self._scope.get_delay() - return 0.0 + return self._scope.get_delay() @delay.setter def delay(self, value: float) -> None: - if self._scope is not None: - self._scope.set_base_delay(value, only_increase=False) + self._scope.set_base_delay(value, only_increase=False) @property def randomize_delay(self) -> bool: - if self._scope is not None: - return bool(self._scope.get_jitter()) - return False + return bool(self._scope.get_jitter()) @property def concurrency(self) -> int: @@ -107,14 +105,10 @@ class _DeprecatedSlotView: category=ScrapyDeprecationWarning, stacklevel=2, ) - if self._scope is not None: - return self._scope.get_concurrency() or 0 - return 0 + return self._scope.get_concurrency() or 0 def free_transfer_slots(self) -> int: - concurrency = ( - self._scope.get_concurrency() or 0 if self._scope is not None else 0 - ) + concurrency = self._scope.get_concurrency() or 0 return concurrency - len(self.transferring) def download_delay(self) -> float: @@ -138,7 +132,9 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): __slots__ = ("_downloader", "_throttler") - def __init__(self, downloader: Downloader, throttler: Any) -> None: + def __init__( + self, downloader: Downloader, throttler: ThrottlingManagerProtocol + ) -> None: self._downloader = downloader self._throttler = throttler @@ -152,11 +148,7 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): def __getitem__(self, key: str) -> _DeprecatedSlotView: if key not in self._active_keys(): raise KeyError(key) - scope = ( - self._throttler.get_scope_manager(key) - if self._throttler is not None - else None - ) + scope = self._throttler.get_scope_manager(key) return _DeprecatedSlotView(self._downloader, key, scope) def __iter__(self) -> Iterator[str]: @@ -193,16 +185,6 @@ class Downloader: category=ScrapyDeprecationWarning, stacklevel=2, ) - for slot_settings in self.per_slot_settings.values(): - for deprecated_key in ("concurrency", "delay", "randomize_delay"): - if deprecated_key in slot_settings: - warnings.warn( - f"The '{deprecated_key}' key in DOWNLOAD_SLOTS is deprecated." - " Use THROTTLING_SCOPES to configure per-domain settings" - " instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) @inlineCallbacks @_warn_spider_arg @@ -230,6 +212,7 @@ class Downloader: category=ScrapyDeprecationWarning, stacklevel=2, ) + assert self.crawler.throttler is not None return _DeprecatedSlotsView(self, self.crawler.throttler) @_warn_spider_arg @@ -237,22 +220,18 @@ class Downloader: self, request: Request, spider: Spider | None = None ) -> tuple[str, _DeprecatedSlotView]: key = self._get_slot_key(request) - scope = ( - self.crawler.throttler.get_scope_manager(key) - if self.crawler.throttler is not None - else None - ) + assert self.crawler.throttler is not None + scope = self.crawler.throttler.get_scope_manager(key) return key, _DeprecatedSlotView(self, key, scope) def _get_slot_key(self, request: Request) -> str: - throttler = self.crawler.throttler - if throttler is not None: - return throttler.get_slot_key(request) - return self.get_slot_key(request) + assert self.crawler.throttler is not None + return self.crawler.throttler.get_slot_key(request) def get_slot_key(self, request: Request) -> str: - # This fallback is used only when no throttler is set; it mirrors the - # historical keying (an explicit download_slot wins, else the domain). + # Retained as public backward-compatible API. It mirrors the historical + # keying (an explicit download_slot wins, else the domain); the slot key + # used at run time comes from the throttler (see _get_slot_key()). meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) if meta_slot is not None: return meta_slot diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index ee76d3786..db3d97444 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -73,10 +73,10 @@ class AutoThrottle: self, response: Response, request: Request, spider: Spider ) -> None: throttler = self.crawler.throttler + assert throttler is not None latency = request.meta.get("download_latency") if ( latency is None - or throttler is None or request.meta.get("autothrottle_dont_adjust_delay", False) is True ): return diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a476602e9..8aac29b7b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -16,7 +16,6 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy import Request - from scrapy.core.downloader import Downloader from scrapy.crawler import Crawler from scrapy.throttling import ScopeID, ThrottlingManagerProtocol @@ -271,22 +270,17 @@ class ScrapyPriorityQueue: class DownloaderInterface: def __init__(self, crawler: Crawler): - assert crawler.engine - self.downloader: Downloader = crawler.engine.downloader - self._throttler: ThrottlingManagerProtocol | None = crawler.throttler + assert crawler.throttler is not None + self._throttler: ThrottlingManagerProtocol = crawler.throttler def stats(self, possible_slots: Iterable[str]) -> list[tuple[float, str]]: return [(self._slot_load(slot), slot) for slot in possible_slots] def get_slot_key(self, request: Request) -> str: - if self._throttler is not None: - return self._throttler.get_slot_key(request) - return self.downloader.get_slot_key(request) + return self._throttler.get_slot_key(request) def _slot_load(self, slot: str) -> float: - if self._throttler is not None: - return self._throttler.get_scope_load(slot) - return 0.0 + return self._throttler.get_scope_load(slot) class DownloaderAwarePriorityQueue: diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5540c2e98..80b204e4d 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -56,7 +56,6 @@ async def test_concurrency_key_deprecated(): downloader = Downloader(crawler) messages = [str(w.message) for w in warns] assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) - assert any("'concurrency' key in DOWNLOAD_SLOTS" in m for m in messages) downloader._get_slot(Request("https://example.com")) downloader.close() @@ -115,7 +114,6 @@ async def test_delay_deprecated(): downloader = Downloader(crawler) messages = [str(w.message) for w in warns] assert any("DOWNLOAD_SLOTS setting is deprecated" in m for m in messages) - assert any("'delay' key in DOWNLOAD_SLOTS" in m for m in messages) downloader._get_slot(Request("https://example.com")) downloader.close() From 3f8841efa3e9ac6358bab2f0b29db246f191e921 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 10:37:22 +0200 Subject: [PATCH 39/55] WIP --- scrapy/core/downloader/__init__.py | 55 +++++++++++++++++++++++++++++- scrapy/throttling.py | 12 ++++--- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 8e4053a92..7e570ace5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,8 +2,10 @@ from __future__ import annotations import random import warnings +from collections import deque from collections.abc import Iterator, Mapping from dataclasses import dataclass, field +from datetime import datetime from typing import TYPE_CHECKING, Any from twisted.internet.defer import inlineCallbacks @@ -30,15 +32,46 @@ if TYPE_CHECKING: ThrottlingManagerProtocol, ThrottlingScopeManagerProtocol, ) + from scrapy.utils.asyncio import CallLaterResult @dataclass(slots=True, eq=False) class _Slot: """Downloader slot""" + concurrency: int + delay: float + randomize_delay: bool + active: set[Request] = field(default_factory=set, init=False, repr=False) + queue: deque[tuple[Request, Deferred[Response]]] = field( + default_factory=deque, init=False, repr=False + ) transferring: set[Request] = field(default_factory=set, init=False, repr=False) lastseen: float = field(default=0, init=False, repr=False) + latercall: CallLaterResult | None = field(default=None, init=False, repr=False) + + def free_transfer_slots(self) -> int: + return self.concurrency - len(self.transferring) + + def download_delay(self) -> float: + if self.randomize_delay: + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311 + return self.delay + + def close(self) -> None: + if self.latercall: + self.latercall.cancel() + self.latercall = None + + def __str__(self) -> str: + return ( + f"" + ) Slot = create_deprecated_class( @@ -205,6 +238,26 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency + @property + def domain_concurrency(self) -> int: + warnings.warn( + "Downloader.domain_concurrency is deprecated. Per-domain concurrency " + "limits are now managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + + @property + def randomize_delay(self) -> bool: + warnings.warn( + "Downloader.randomize_delay is deprecated. Delay randomization is now " + "managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") + @property def slots(self) -> _DeprecatedSlotsView: warnings.warn( @@ -235,7 +288,7 @@ class Downloader: meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) if meta_slot is not None: return meta_slot - return urlparse_cached(request).netloc or "" + return urlparse_cached(request).hostname or "" async def _enqueue_request(self, request: Request) -> Response: key = self._get_slot_key(request) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index cc38a0d47..c8a910e83 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -473,7 +473,7 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: class MyThrottlingManager: @scope_cache async def get_scopes(self, request): - return urlparse_cached(request).netloc + return urlparse_cached(request).hostname or "" """ @wraps(f) @@ -542,8 +542,8 @@ class ThrottlingManager: :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLING_SCOPES`. Each ``DOWNLOAD_SLOTS`` entry is translated to a throttling scope keyed - by the same slot name (the default manager keys domain scopes by - ``netloc``, which is what download slots used too): ``concurrency`` and + by the same slot name (the default manager keys domain scopes by host + name, which is what download slots used too): ``concurrency`` and ``delay`` map directly, and the ``randomize_delay`` boolean maps to a ``jitter`` magnitude (the historical ±50%, or none). An explicit ``THROTTLING_SCOPES`` entry for the same scope takes precedence over the @@ -592,7 +592,7 @@ class ThrottlingManager: stacklevel=2, ) return cast("RequestScopes", download_slot) - return urlparse_cached(request).netloc + return urlparse_cached(request).hostname or "" def get_slot_key(self, request: Request) -> str: scopes = self._resolve_scopes_sync(request) @@ -852,7 +852,9 @@ class ThrottlingManager: except Exception: # pragma: no cover - backend-specific failures return if delay: - self.apply_robots_crawl_delay(urlparse_cached(request).netloc, delay) + self.apply_robots_crawl_delay( + urlparse_cached(request).hostname or "", delay + ) def apply_robots_crawl_delay(self, scope_id: ScopeID, delay: float) -> None: """Honor a robots.txt ``Crawl-delay`` directive of *delay* seconds for From 370ff74dc91e949ea7600584b13e93a2d43efe56 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 15:02:13 +0200 Subject: [PATCH 40/55] WIP --- docs/topics/throttling.rst | 9 +++++++++ scrapy/downloadermiddlewares/backoff.py | 6 ++++++ scrapy/extensions/throttle.py | 5 ++++- scrapy/pqueues.py | 4 ++-- scrapy/settings/default_settings.py | 2 ++ 5 files changed, 23 insertions(+), 3 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index accf6a590..61d7198a4 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -949,6 +949,15 @@ its domain and its IP, and is only sent when **both** allow it (see Additional settings =================== +- .. setting:: BACKOFF_ENABLED + + :setting:`BACKOFF_ENABLED` (default: ``True``) + + Whether to enable the :class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, + which drives :ref:`backoff ` from download outcomes. Set it to + ``False`` to disable backoff without having to remove the middleware from + :setting:`DOWNLOADER_MIDDLEWARES`. + - .. setting:: BACKOFF_EXCEPTIONS :setting:`BACKOFF_EXCEPTIONS` diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index da6f2a9f1..0bc45ed7c 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -5,6 +5,7 @@ import logging from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING +from scrapy.exceptions import NotConfigured from scrapy.throttling import iter_scopes from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import load_object @@ -69,10 +70,15 @@ class BackoffMiddleware: ` to back off the request's scopes through its :meth:`~scrapy.throttling.ThrottlingManagerProtocol.back_off` API. + It is enabled by default; set :setting:`BACKOFF_ENABLED` to ``False`` to + disable it without removing it from :setting:`DOWNLOADER_MIDDLEWARES`. + See :ref:`throttling` for details. """ def __init__(self, crawler: Crawler): + if not crawler.settings.getbool("BACKOFF_ENABLED"): + raise NotConfigured # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a # non-None default and is instantiated before the downloader is built, # so crawler.throttler is always set here (the engine likewise asserts diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index db3d97444..5a5bd540f 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -83,7 +83,10 @@ class AutoThrottle: # AutoThrottle predates throttling scopes, so it adjusts the delay of # the request's domain scope, matching its historical per-domain slots. - scope_id = urlparse_cached(request).netloc + # Key by hostname (not netloc) to match the default ThrottlingManager + # scope, so the adjusted delay applies to the scope actually enforced + # even for non-default ports. + scope_id = urlparse_cached(request).hostname or "" olddelay = self._scope_delay(throttler, scope_id) newdelay = self._adjust_delay(olddelay, latency, response) throttler.set_scope_delay(scope_id, newdelay) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 8aac29b7b..a767c9dbd 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -268,7 +268,7 @@ class ScrapyPriorityQueue: ) -class DownloaderInterface: +class _DownloaderInterface: def __init__(self, crawler: Crawler): assert crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = crawler.throttler @@ -347,7 +347,7 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler) + self._downloader_interface: _DownloaderInterface = _DownloaderInterface(crawler) self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index e4f9788d4..23ff40b9c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -33,6 +33,7 @@ __all__ = [ "AWS_USE_SSL", "AWS_VERIFY", "BACKOFF_DELAY_FACTOR", + "BACKOFF_ENABLED", "BACKOFF_EXCEPTIONS", "BACKOFF_HTTP_CODES", "BACKOFF_JITTER", @@ -255,6 +256,7 @@ AWS_USE_SSL = None AWS_VERIFY = None BACKOFF_DELAY_FACTOR = 2.0 +BACKOFF_ENABLED = True BACKOFF_EXCEPTIONS = [ "scrapy.exceptions.DownloadFailedError", "scrapy.exceptions.DownloadTimeoutError", From 584462815d2843d5002a616a6e7da587c8a0cadf Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 2 Jul 2026 16:20:51 +0200 Subject: [PATCH 41/55] Improve the default warning about concurrency settings --- scrapy/throttling.py | 28 ++++++++++++++++++++++++---- tests/test_crawler.py | 4 ++-- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index c8a910e83..b27bd9319 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -168,7 +168,14 @@ def _default_scope_concurrency(settings: BaseSettings) -> int: def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: """Warn about the concurrency settings bridged by :func:`_default_scope_concurrency`. Call once per crawl (see - :meth:`ThrottlingManager.__init__`).""" + :meth:`ThrottlingManager.__init__`). + + When :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is set explicitly, warn that + it is deprecated. When neither concurrency setting is set explicitly, warn + that the effective per-scope concurrency is still the deprecated setting's + value (kept for backward compatibility) and will drop to + :setting:`THROTTLING_SCOPE_CONCURRENCY`'s default once the deprecated + setting is removed, so users can pin it explicitly.""" default_priority = SETTINGS_PRIORITIES["default"] domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") @@ -184,10 +191,23 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: elif not scope_set: current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") future = settings.getint("THROTTLING_SCOPE_CONCURRENCY") + # This warning only makes sense while the two defaults differ. If the + # default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN is lowered to + # match THROTTLING_SCOPE_CONCURRENCY before this branch is merged, the + # message becomes nonsensical ("will change from 1 to 1"); fail loudly + # here so it is revisited rather than shipping a bogus warning. + assert current != future, ( + "CONCURRENT_REQUESTS_PER_DOMAIN and THROTTLING_SCOPE_CONCURRENCY now " + "share the same default; drop this warn-then-flip warning." + ) warnings.warn( - f"The default value of THROTTLING_SCOPE_CONCURRENCY will change " - f"from {current} to {future} in a future Scrapy version. Set " - f"THROTTLING_SCOPE_CONCURRENCY explicitly to silence this warning.", + f"The effective per-scope (per-domain) concurrency is {current}, " + f"the default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN " + f"setting, which is still respected for backward compatibility. " + f"Once CONCURRENT_REQUESTS_PER_DOMAIN is removed, it will drop to " + f"{future}, the default of THROTTLING_SCOPE_CONCURRENCY. Set " + f"THROTTLING_SCOPE_CONCURRENCY explicitly to choose a value and " + f"silence this warning.", category=ScrapyDeprecationWarning, stacklevel=2, ) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index d68d63392..578b2396a 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -93,7 +93,7 @@ class TestScopeConcurrencyBridge: def test_warns_flip_when_neither_set(self) -> None: with pytest.warns( ScrapyDeprecationWarning, - match="THROTTLING_SCOPE_CONCURRENCY will change", + match="Once CONCURRENT_REQUESTS_PER_DOMAIN is removed", ): get_crawler(prevent_warnings=False) @@ -116,7 +116,7 @@ class TestScopeConcurrencyBridge: ) messages = [str(w.message) for w in recorded] assert not any( - "THROTTLING_SCOPE_CONCURRENCY will change" in m for m in messages + "Once CONCURRENT_REQUESTS_PER_DOMAIN is removed" in m for m in messages ) assert not any( "CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated" in m From 5b276a3da1a120e3fee95f0201d1bd7646a29faa Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 05:32:31 +0200 Subject: [PATCH 42/55] KISS --- docs/topics/throttling.rst | 18 ++---- scrapy/crawler.py | 78 ++++++++----------------- scrapy/downloadermiddlewares/backoff.py | 13 ++--- scrapy/throttling.py | 13 ++++- 4 files changed, 42 insertions(+), 80 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 61d7198a4..f6da9f615 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -667,9 +667,7 @@ Alternative approaches include: @scope_cache async def get_scopes(self, request): extracted = tldextract.extract(request.url) - if extracted.domain and extracted.suffix: - return f"{extracted.domain}.{extracted.suffix}" - return urlparse_cached(request).netloc + return extracted.registered_domain or urlparse_cached(request).netloc THROTTLING_MANAGER = MyThrottlingManager @@ -698,18 +696,10 @@ Alternative approaches include: @scope_cache async def get_scopes(self, request): extracted = tldextract.extract(request.url) - if not (extracted.domain and extracted.suffix): + if not extracted.registered_domain: return urlparse_cached(request).netloc - scopes = set() - registrable_domain = f"{extracted.domain}.{extracted.suffix}" - scopes.add(registrable_domain) - if extracted.subdomain: - subdomain_parts = extracted.subdomain.split(".") - for i in range(len(subdomain_parts)): - subdomain = ".".join(subdomain_parts[i:]) - full_domain = f"{subdomain}.{registrable_domain}" - scopes.add(full_domain) - return scopes + # The registrable domain, plus the full host for a subdomain. + return {extracted.registered_domain, extracted.fqdn} THROTTLING_MANAGER = MyThrottlingManager diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 84f6774d5..e5df6f15f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -104,8 +104,14 @@ class Crawler: return self.addons.load_settings(self.settings) - self._apply_spider_download_delay() - self._apply_spider_max_concurrent_requests() + self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") + # 'max_concurrent_requests' historically overrode the per-domain slot + # concurrency, which is now THROTTLING_SCOPE_CONCURRENCY (see + # scrapy.throttling._default_scope_concurrency); the old deprecation + # message pointed at CONCURRENT_REQUESTS, which never matched that. + self._apply_deprecated_spider_attr( + "max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY" + ) self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) @@ -165,65 +171,29 @@ class Crawler: "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)} ) - def _apply_spider_download_delay(self) -> None: + def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None: + """Bridge a deprecated spider attribute onto *setting*, warning about + the deprecation (and about being ignored when *setting* is already set + at spider or higher priority).""" spider = self.spider if self.spider is not None else self.spidercls - if not hasattr(spider, "download_delay"): + if not hasattr(spider, attr): return - delay_prio = self.settings.getpriority("DOWNLOAD_DELAY") or 0 - if delay_prio >= SETTINGS_PRIORITIES["spider"]: + if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]: warnings.warn( - "The 'download_delay' spider attribute is deprecated. " - "It is also being ignored because DOWNLOAD_DELAY is already set " - "at spider or higher priority. Remove the 'download_delay' " - "attribute from your spider.", + f"The {attr!r} spider attribute is deprecated. It is also being " + f"ignored because {setting} is already set at spider or higher " + f"priority. Remove the {attr!r} attribute from your spider.", category=ScrapyDeprecationWarning, - stacklevel=2, + stacklevel=3, ) - else: - warnings.warn( - "The 'download_delay' spider attribute is deprecated. Use the " - "DOWNLOAD_DELAY setting or per-domain THROTTLING_SCOPES instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.settings.set( - "DOWNLOAD_DELAY", spider.download_delay, priority="spider" - ) - - def _apply_spider_max_concurrent_requests(self) -> None: - spider = self.spider if self.spider is not None else self.spidercls - if not hasattr(spider, "max_concurrent_requests"): return - # Historically this attribute overrode the per-domain slot concurrency, - # which is now THROTTLING_SCOPE_CONCURRENCY (see - # scrapy.throttling._default_scope_concurrency). The old deprecation - # message pointed at CONCURRENT_REQUESTS, but that never matched its - # actual per-domain effect. - concurrency_prio = ( - self.settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") or 0 + warnings.warn( + f"The {attr!r} spider attribute is deprecated. Use the {setting} " + f"setting or per-domain THROTTLING_SCOPES instead.", + category=ScrapyDeprecationWarning, + stacklevel=3, ) - if concurrency_prio >= SETTINGS_PRIORITIES["spider"]: - warnings.warn( - "The 'max_concurrent_requests' spider attribute is deprecated. " - "It is also being ignored because THROTTLING_SCOPE_CONCURRENCY is " - "already set at spider or higher priority. Remove the " - "'max_concurrent_requests' attribute from your spider.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - else: - warnings.warn( - "The 'max_concurrent_requests' spider attribute is deprecated. Use " - "the THROTTLING_SCOPE_CONCURRENCY setting or per-domain " - "THROTTLING_SCOPES instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.settings.set( - "THROTTLING_SCOPE_CONCURRENCY", - spider.max_concurrent_requests, - priority="spider", - ) + self.settings.set(setting, getattr(spider, attr), priority="spider") def _apply_reactorless_default_settings(self) -> None: """Change some setting defaults when not using a Twisted reactor. diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 0bc45ed7c..803143d34 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -6,9 +6,8 @@ from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured -from scrapy.throttling import iter_scopes +from scrapy.throttling import _load_exceptions, iter_scopes from scrapy.utils.decorators import _warn_spider_arg -from scrapy.utils.misc import load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -94,19 +93,15 @@ class BackoffMiddleware: self._http_codes: set[int] = { int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") } - self._exceptions: tuple[type[BaseException], ...] = tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in settings.getlist("BACKOFF_EXCEPTIONS") + self._exceptions: tuple[type[BaseException], ...] = _load_exceptions( + settings.getlist("BACKOFF_EXCEPTIONS") ) for scope_config in settings.getdict("THROTTLING_SCOPES").values(): backoff = scope_config.get("backoff") or {} if "http_codes" in backoff: self._http_codes.update(int(code) for code in backoff["http_codes"]) if "exceptions" in backoff: - self._exceptions += tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in backoff["exceptions"] - ) + self._exceptions += _load_exceptions(backoff["exceptions"]) @classmethod def from_crawler(cls, crawler: Crawler) -> Self: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index b27bd9319..19fc66a61 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -132,6 +132,14 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None +def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], ...]: + """Resolve *exceptions* (exception classes or their import paths) to a tuple + of exception classes.""" + return tuple( + load_object(exc) if isinstance(exc, str) else exc for exc in exceptions + ) + + def _default_scope_concurrency(settings: BaseSettings) -> int: """Return the default concurrency of a throttling scope that does not set its own ``concurrency``. @@ -1186,9 +1194,8 @@ class ThrottlingScopeManager: "http_codes", settings.getlist("BACKOFF_HTTP_CODES") ) } - self._backoff_exceptions: tuple[type[BaseException], ...] = tuple( - load_object(exc) if isinstance(exc, str) else exc - for exc in backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) + self._backoff_exceptions: tuple[type[BaseException], ...] = _load_exceptions( + backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) ) self._window: float = settings.getfloat("BACKOFF_WINDOW") From 6dfbe1e30bd18d5305d95a0402d115e414e706c2 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 06:16:55 +0200 Subject: [PATCH 43/55] KISS --- docs/intro/tutorial.rst | 23 ++++ scrapy/core/downloader/__init__.py | 16 ++- .../templates/project/module/settings.py.tmpl | 11 +- scrapy/throttling.py | 115 +++++++++--------- tests/test_throttling.py | 13 ++ 5 files changed, 109 insertions(+), 69 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index c4e04364b..d61b373ba 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -130,6 +130,29 @@ and defines some attributes and methods: the scraped data as dicts and also finding new URLs to follow and creating new requests (:class:`~scrapy.Request`) from them. +Crawling faster for this tutorial +--------------------------------- + +By default, Scrapy :ref:`throttles ` crawls to be polite: it sends +at most one request at a time to each website, waiting one second between +requests. That keeps you from overwhelming websites, but it also makes crawling +slow, and it means the requests above are sent one after another rather than in +parallel. + +``quotes.toscrape.com`` is a sandbox meant for scraping practice, so we can +safely crawl it faster. Open ``tutorial/settings.py`` and add a +:setting:`THROTTLING_SCOPES` entry that raises the concurrency and lowers the +delay for that domain only: + +.. code-block:: python + + THROTTLING_SCOPES = { + "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, + } + +The polite defaults still apply to every other website. Remove this entry once +you move on to crawling your own sites. + How to run our spider --------------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7e570ace5..5394926f1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -114,13 +114,19 @@ class _DeprecatedSlotView: if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key } + # This deprecated view reads throttling scope state from private attributes + # of the default scope manager rather than through the scope manager + # protocol: these are read-only compatibility accessors, so keeping them off + # the protocol avoids forcing custom THROTTLING_SCOPE_MANAGER implementations + # to provide members that only exist to feed this shim. A custom manager that + # lacks the attribute simply falls back to the historical default. @property def lastseen(self) -> float: - return 0.0 + return getattr(self._scope, "_last_seen", None) or 0.0 @property def delay(self) -> float: - return self._scope.get_delay() + return getattr(self._scope, "_delay", 0.0) @delay.setter def delay(self, value: float) -> None: @@ -128,7 +134,7 @@ class _DeprecatedSlotView: @property def randomize_delay(self) -> bool: - return bool(self._scope.get_jitter()) + return bool(getattr(self._scope, "_jitter", None)) @property def concurrency(self) -> int: @@ -138,10 +144,10 @@ class _DeprecatedSlotView: category=ScrapyDeprecationWarning, stacklevel=2, ) - return self._scope.get_concurrency() or 0 + return getattr(self._scope, "_concurrency", None) or 0 def free_transfer_slots(self) -> int: - concurrency = self._scope.get_concurrency() or 0 + concurrency = getattr(self._scope, "_concurrency", None) or 0 return concurrency - len(self.transferring) def download_delay(self) -> float: diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index a035cc9ed..b9faac9a5 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -16,12 +16,11 @@ ROBOTSTXT_OBEY = True THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 -# Crawl the tutorial websites faster, overriding the polite defaults above only -# for these domains (remove this once you crawl your own sites): -THROTTLING_SCOPES = { - "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, - "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, -} +# Override the polite defaults above for specific domains, e.g. to crawl a site +# you own (or one meant for scraping) faster: +#THROTTLING_SCOPES = { +# "example.com": {"concurrency": 16, "delay": 0.1}, +#} # Set settings whose default value is deprecated to a future-proof value: FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 19fc66a61..f782bae42 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -104,13 +104,7 @@ def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: this helper normalizes any of those into an iterable of scope IDs, e.g. to react to a request's scopes in a custom middleware. """ - if scopes is None: - return () - if isinstance(scopes, str): - return (scopes,) - if isinstance(scopes, dict): - return scopes.keys() - return iter(scopes) + return (scope for scope, _ in iter_scope_values(scopes)) def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | None]]: @@ -197,17 +191,12 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: stacklevel=2, ) elif not scope_set: + # This warn-then-flip message only makes sense while the two defaults + # differ (otherwise it reads "will drop from 1 to 1"). That invariant is + # guarded by test_deprecated_concurrency_defaults_differ rather than at + # run time, so a crawl is never aborted over it. current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") future = settings.getint("THROTTLING_SCOPE_CONCURRENCY") - # This warning only makes sense while the two defaults differ. If the - # default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN is lowered to - # match THROTTLING_SCOPE_CONCURRENCY before this branch is merged, the - # message becomes nonsensical ("will change from 1 to 1"); fail loudly - # here so it is revisited rather than shipping a bogus warning. - assert current != future, ( - "CONCURRENT_REQUESTS_PER_DOMAIN and THROTTLING_SCOPE_CONCURRENCY now " - "share the same default; drop this warn-then-flip warning." - ) warnings.warn( f"The effective per-scope (per-domain) concurrency is {current}, " f"the default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN " @@ -727,9 +716,7 @@ class ThrottlingManager: manager for manager, _ in managers if manager.concurrency_blocked() ] if not blocked: - for manager, value in managers: - manager.record_sent(amount=value) - self._reserved[request] = managers + self._record_reservation(request, managers) return if self._debug: logger.debug( @@ -738,6 +725,18 @@ class ThrottlingManager: ) await self._wait_for_slot(blocked) + def _record_reservation( + self, + request: Request, + managers: list[tuple[ThrottlingScopeManagerProtocol, float | None]], + ) -> None: + """Record a send on each of *request*'s scope *managers* and mark + *request* as reserved, so :meth:`release` can later free the slots. This + is the shared tail of :meth:`acquire` and :meth:`reserve`.""" + for manager, value in managers: + manager.record_sent(amount=value) + self._reserved[request] = managers + def release(self, request: Request) -> None: managers = self._reserved.pop(request, None) if not managers: @@ -769,9 +768,7 @@ class ThrottlingManager: (self.get_scope_manager(scope_id), value) for scope_id, value in self._cached_scope_values(request) ] - for manager, value in managers: - manager.record_sent(amount=value) - self._reserved[request] = managers + self._record_reservation(request, managers) def get_time_until_ready(self, request: Request) -> float | None: now = time.monotonic() @@ -806,14 +803,19 @@ class ThrottlingManager: async def _delay_request(self, request: Request) -> None: """Honor the :reqmeta:`throttling_delay` meta key by holding *request* - for the requested number of seconds the first time it is processed.""" - delay = request.meta.get("throttling_delay") - if not delay or request.meta.get("_throttling_delayed"): + for the requested number of seconds the first time it is processed. + + This is the blocking (:meth:`acquire`) counterpart of + :meth:`_request_delay_deadline`, which the readiness API polls instead; + both share the deadline bookkeeping and the one-time debug log. Here the + deadline is honored by sleeping until it, then marking the delay as + consumed so the request is never held again.""" + now = time.monotonic() + wait = self._request_delay_deadline(request, now) - now + if wait <= 0: return + await sleep(wait) request.meta["_throttling_delayed"] = True - if self._debug: - logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") - await sleep(float(delay)) def _request_delay_deadline(self, request: Request, now: float) -> float: """Return the monotonic time before which *request* must not be sent due @@ -1042,18 +1044,6 @@ class ThrottlingScopeManagerProtocol(Protocol): """Return whether *exception* triggers backoff for this scope (defaults to :setting:`BACKOFF_EXCEPTIONS`).""" - def get_delay(self) -> float: - """Return the current effective delay of this scope, in seconds, - including any active backoff (unlike :meth:`get_base_delay`).""" - - def get_jitter(self) -> float | list[float]: - """Return the magnitude of the random variation applied to the delay - (the per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`).""" - - def get_concurrency(self) -> int | None: - """Return the maximum number of concurrent requests allowed for this - scope, or ``None`` when the scope enforces no explicit limit.""" - def get_base_delay(self) -> float: """Return the base (non-backoff) delay of this scope, in seconds.""" @@ -1167,11 +1157,14 @@ class ThrottlingScopeManager: self._base_delay: float = float( config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) ) - # Magnitude of the random variation applied to the (non-backoff) delay. + # Magnitude of the random variation applied to the (non-backoff) delay, + # normalized to a (low, high) multiplier range (or None for no jitter). # Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay # randomization is on, or to no variation when it is off. - self._jitter: float | list[float] = config.get( - "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 + self._jitter: tuple[float, float] | None = self._normalize_jitter( + config.get( + "jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0 + ) ) self._delay_factor: float = float( backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR")) @@ -1182,8 +1175,8 @@ class ThrottlingScopeManager: self._min_delay: float = float( backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY")) ) - self._backoff_jitter: float | list[float] = backoff.get( - "jitter", settings.getfloat("BACKOFF_JITTER") + self._backoff_jitter: tuple[float, float] | None = self._normalize_jitter( + backoff.get("jitter", settings.getfloat("BACKOFF_JITTER")) ) # Which responses/exceptions trigger backoff for this scope. Each # defaults to the matching global BACKOFF_* setting (see @@ -1252,13 +1245,28 @@ class ThrottlingScopeManager: return time.monotonic() if now is None else now @staticmethod - def _apply_jitter(value: float, jitter: float | list[float]) -> float: + def _normalize_jitter( + jitter: float | list[float], + ) -> tuple[float, float] | None: + """Normalize a ``jitter`` config value to a ``(low, high)`` multiplier + range, or ``None`` when no jitter applies. + + A scalar ``j`` means the symmetric range ``(-j, +j)``, so that + ``value * (1 + uniform(-j, +j))`` matches the historical + ``value * uniform(1 - j, 1 + j)``; a list/tuple is taken as an explicit + ``[low, high]`` range. + """ if isinstance(jitter, (list, tuple)): - low, high = jitter[0], jitter[1] - return value * (1 + random.uniform(low, high)) # noqa: S311 + return (float(jitter[0]), float(jitter[1])) if not jitter: + return None + return (-float(jitter), float(jitter)) + + @staticmethod + def _apply_jitter(value: float, jitter: tuple[float, float] | None) -> float: + if jitter is None: return value - return value * random.uniform(1 - jitter, 1 + jitter) # noqa: S311 + return value * (1 + random.uniform(*jitter)) # noqa: S311 def _effective_delay(self) -> float: # ``self._delay`` is the deterministic delay (the base delay, or the @@ -1474,15 +1482,6 @@ class ThrottlingScopeManager: def triggers_backoff_for_exception(self, exception: BaseException) -> bool: return isinstance(exception, self._backoff_exceptions) - def get_delay(self) -> float: - return self._delay - - def get_jitter(self) -> float | list[float]: - return self._jitter - - def get_concurrency(self) -> int | None: - return self._concurrency - def get_base_delay(self) -> float: return self._base_delay diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 72cabaa0c..3dacbe0ef 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -7,6 +7,7 @@ import pytest from scrapy import signals from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response +from scrapy.settings import default_settings from scrapy.throttling import ( ThrottlingManager, ThrottlingScopeManager, @@ -58,6 +59,18 @@ def _response(status=200, headers=None, url="http://example.com", meta=None): return Response(url, status=status, headers=headers or {}, request=request) +def test_deprecated_concurrency_defaults_differ(): + """``_warn_on_deprecated_concurrency`` emits a warn-then-flip message that + only makes sense while the two concurrency defaults differ (otherwise it + reads "will drop from N to N"). Guard that invariant here so that lowering + the deprecated default to match is caught by the test suite instead of + shipping a bogus warning or aborting a crawl.""" + assert ( + default_settings.CONCURRENT_REQUESTS_PER_DOMAIN + != default_settings.THROTTLING_SCOPE_CONCURRENCY + ) + + class TestThrottlingManager: @coroutine_test async def test_get_scopes_returns_netloc(self): From 1f9fbee5a4afee07c41b5ec52fbe143facd3f27d Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 06:44:28 +0200 Subject: [PATCH 44/55] KISS --- scrapy/pqueues.py | 27 ++++--------- scrapy/throttling.py | 95 ++++++++++++++++++++------------------------ 2 files changed, 52 insertions(+), 70 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index a767c9dbd..69a0e7ea9 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -268,21 +268,6 @@ class ScrapyPriorityQueue: ) -class _DownloaderInterface: - def __init__(self, crawler: Crawler): - assert crawler.throttler is not None - self._throttler: ThrottlingManagerProtocol = crawler.throttler - - def stats(self, possible_slots: Iterable[str]) -> list[tuple[float, str]]: - return [(self._slot_load(slot), slot) for slot in possible_slots] - - def get_slot_key(self, request: Request) -> str: - return self._throttler.get_slot_key(request) - - def _slot_load(self, slot: str) -> float: - return self._throttler.get_scope_load(slot) - - class DownloaderAwarePriorityQueue: """PriorityQueue which takes Downloader activity into account: domains (slots) with the least amount of active downloads are dequeued @@ -347,7 +332,8 @@ class DownloaderAwarePriorityQueue: "queue class can be resumed." ) - self._downloader_interface: _DownloaderInterface = _DownloaderInterface(crawler) + assert crawler.throttler is not None + self._throttler: ThrottlingManagerProtocol = crawler.throttler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key @@ -397,8 +383,11 @@ class DownloaderAwarePriorityQueue: start_queue_cls=self._start_queue_cls, ) + def _slot_stats(self) -> list[tuple[float, str]]: + return [(self._throttler.get_scope_load(slot), slot) for slot in self.pqueues] + def pop(self) -> Request | None: - stats = self._downloader_interface.stats(self.pqueues) + stats = self._slot_stats() if not stats: return None @@ -411,7 +400,7 @@ class DownloaderAwarePriorityQueue: return request def push(self, request: Request) -> None: - slot = self._downloader_interface.get_slot_key(request) + slot = self._throttler.get_slot_key(request) if slot not in self.pqueues: self.pqueues[slot] = self.pqfactory(slot) queue = self.pqueues[slot] @@ -424,7 +413,7 @@ class DownloaderAwarePriorityQueue: Raises :exc:`NotImplementedError` if the underlying queue class does not implement a ``peek`` method, which is optional for queues. """ - stats = self._downloader_interface.stats(self.pqueues) + stats = self._slot_stats() if not stats: return None slot = self._next_slot(stats, update_state=False) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index f782bae42..480bb3a8f 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -134,6 +134,14 @@ def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], .. ) +def _effective_priority(settings: BaseSettings, name: str) -> int: + """Return the priority of setting *name*, treating an unset setting (no + priority, ``None``) as just below ``"default"`` so it never wins over one + that is at least at its default value.""" + priority = settings.getpriority(name) + return SETTINGS_PRIORITIES["default"] - 1 if priority is None else priority + + def _default_scope_concurrency(settings: BaseSettings) -> int: """Return the default concurrency of a throttling scope that does not set its own ``concurrency``. @@ -147,14 +155,8 @@ def _default_scope_concurrency(settings: BaseSettings) -> int: :func:`_warn_on_deprecated_concurrency`). """ default_priority = SETTINGS_PRIORITIES["default"] - # An unset setting has no priority (``None``); treat it as below "default" - # so it never wins over one that is at least at its default value. - domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") - domain_priority = ( - default_priority - 1 if domain_priority is None else domain_priority - ) - scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") - scope_priority = default_priority - 1 if scope_priority is None else scope_priority + domain_priority = _effective_priority(settings, "CONCURRENT_REQUESTS_PER_DOMAIN") + scope_priority = _effective_priority(settings, "THROTTLING_SCOPE_CONCURRENCY") if domain_priority > scope_priority: return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") if scope_priority > domain_priority: @@ -179,10 +181,13 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: :setting:`THROTTLING_SCOPE_CONCURRENCY`'s default once the deprecated setting is removed, so users can pin it explicitly.""" default_priority = SETTINGS_PRIORITIES["default"] - domain_priority = settings.getpriority("CONCURRENT_REQUESTS_PER_DOMAIN") - scope_priority = settings.getpriority("THROTTLING_SCOPE_CONCURRENCY") - domain_set = domain_priority is not None and domain_priority > default_priority - scope_set = scope_priority is not None and scope_priority > default_priority + domain_set = ( + _effective_priority(settings, "CONCURRENT_REQUESTS_PER_DOMAIN") + > default_priority + ) + scope_set = ( + _effective_priority(settings, "THROTTLING_SCOPE_CONCURRENCY") > default_priority + ) if domain_set: warnings.warn( "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " @@ -210,39 +215,21 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: ) -def _to_scope_dict(collection: Any, default: Callable[[], Any]) -> dict[ScopeID, Any]: - """Normalize *collection* (``None``, str, iterable or dict) into a dict - mapping scope names to values produced by *default*.""" - if isinstance(collection, dict): - return collection - if collection is None: +def _to_scope_dict(scopes: RequestScopes) -> dict[ScopeID, float | None]: + """Normalize *scopes* (``None``, a scope id, an iterable of scope ids or a + ``{scope_id: quota}`` dict) into a ``{scope_id: quota}`` dict, using ``None`` + as the quota of scopes that have none.""" + if isinstance(scopes, dict): + return scopes + if scopes is None: return {} - if isinstance(collection, str): - return {collection: default()} - if isinstance(collection, Iterable): - return {scope: default() for scope in collection} + if isinstance(scopes, str): + return {scopes: None} + if isinstance(scopes, Iterable): + return dict.fromkeys(scopes) raise TypeError( - f"Invalid type ({type(collection)}) of scopes value " - f"{collection!r}. Expected None, str, Iterable or dict." - ) - - -def _add_bare_scope(collection: Any, scope: ScopeID, empty: Any) -> Any: - """Add *scope* to *collection* without any associated value, keeping the - most compact representation possible.""" - if collection is None: - return scope - if isinstance(collection, str): - return collection if collection == scope else {collection, scope} - if isinstance(collection, dict): - if scope not in collection: - collection[scope] = empty - return collection - if isinstance(collection, Iterable): - return set(collection) | {scope} if scope not in collection else collection - raise TypeError( - f"Invalid type ({type(collection)}) of scopes value " - f"{collection!r}. Expected None, str, Iterable or dict." + f"Invalid type ({type(scopes)}) of scopes value " + f"{scopes!r}. Expected None, str, Iterable or dict." ) @@ -251,20 +238,26 @@ def add_scope( scope: ScopeID, value: float | None = None, /, -) -> RequestScopes: - """Add *scope* to *scopes* with *value*. +) -> dict[ScopeID, float | None]: + """Add *scope* to *scopes* with *value*, returning a ``{scope_id: quota}`` + dict. This is a utility function to help extending the output of :meth:`~ThrottlingManagerProtocol.get_scopes`, e.g. in :class:`ThrottlingManager` subclasses. + + Adding a scope with a *value* fails if it is already present, so an existing + :ref:`quota ` is never silently overwritten; adding it + without a value leaves any existing entry untouched. """ + result = _to_scope_dict(scopes) if value is None: - return cast("RequestScopes", _add_bare_scope(scopes, scope, None)) - scopes = _to_scope_dict(scopes, lambda: None) - if scope in scopes and not isinstance(scopes[scope], dict): - raise TypeError(f"Scope {scope!r} has a non-dict value in {scopes!r}") - scopes[scope] = value - return scopes + result.setdefault(scope, None) + return result + if scope in result: + raise TypeError(f"Scope {scope!r} already has a value in {scopes!r}") + result[scope] = value + return result class ThrottlingManagerProtocol(Protocol): From c8e9fbd7a1ddd5ef175a9b913ad45f7e9755e9f3 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 07:13:46 +0200 Subject: [PATCH 45/55] KISS --- scrapy/core/downloader/__init__.py | 9 --------- scrapy/downloadermiddlewares/backoff.py | 23 +++++++++++++---------- scrapy/settings/default_settings.py | 1 - 3 files changed, 13 insertions(+), 20 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 5394926f1..a28333c90 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -274,15 +274,6 @@ class Downloader: assert self.crawler.throttler is not None return _DeprecatedSlotsView(self, self.crawler.throttler) - @_warn_spider_arg - def _get_slot( - self, request: Request, spider: Spider | None = None - ) -> tuple[str, _DeprecatedSlotView]: - key = self._get_slot_key(request) - assert self.crawler.throttler is not None - scope = self.crawler.throttler.get_scope_manager(key) - return key, _DeprecatedSlotView(self, key, scope) - def _get_slot_key(self, request: Request) -> str: assert self.crawler.throttler is not None return self.crawler.throttler.get_slot_key(request) diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 803143d34..54b429fb7 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -22,14 +22,22 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _parse_retry_after(response: Response) -> float | None: - raw = response.headers.get("Retry-After") +def _decoded_header(response: Response, name: str) -> str | None: + """Return the stripped UTF-8 value of the *name* header of *response*, or + ``None`` if it is absent or not valid UTF-8.""" + raw = response.headers.get(name) if not raw: return None try: - value = raw.decode("utf-8").strip() + return raw.decode("utf-8").strip() except UnicodeDecodeError: return None + + +def _parse_retry_after(response: Response) -> float | None: + value = _decoded_header(response, "Retry-After") + if value is None: + return None if value.isdigit(): return float(value) # seconds try: @@ -46,12 +54,8 @@ def _parse_retry_after(response: Response) -> float | None: def _parse_ratelimit_reset(response: Response) -> float | None: - raw = response.headers.get("RateLimit-Reset") - if not raw: - return None - try: - value = raw.decode("utf-8").strip() - except UnicodeDecodeError: + value = _decoded_header(response, "RateLimit-Reset") + if value is None: return None try: return float(value) @@ -151,7 +155,6 @@ class BackoffMiddleware: ] if matched: self._throttler.back_off(matched) - return @staticmethod def _response_delay(response: Response) -> float | None: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 23ff40b9c..b9b9d19f3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -268,7 +268,6 @@ BACKOFF_MAX_DELAY = 300.0 BACKOFF_MIN_DELAY = 1.0 BACKOFF_WINDOW = 60.0 - BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 From 555e8736f2e7823492aee9ea4638d989b8b91e6e Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 07:39:00 +0200 Subject: [PATCH 46/55] KISS --- docs/topics/throttling.rst | 10 +++--- scrapy/core/downloader/__init__.py | 3 -- scrapy/downloadermiddlewares/backoff.py | 44 +----------------------- scrapy/downloadermiddlewares/redirect.py | 2 +- scrapy/throttling.py | 9 ++--- 5 files changed, 10 insertions(+), 58 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index f6da9f615..bfa126fc8 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -75,15 +75,15 @@ behavior for specific domains [1]_. It is a dict that maps scope names to :class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. It is empty by default. -:command:`startproject` scaffolds an entry for the testing websites used during -the :ref:`tutorial `, so that they are crawled faster while the -:ref:`conservative defaults ` still apply to other domains: +:command:`startproject` scaffolds a commented-out example entry, so that you can +uncomment and edit it to crawl domains you own (or that are meant for scraping) +faster, while the :ref:`conservative defaults ` still apply to +other domains: .. code-block:: python THROTTLING_SCOPES = { - "books.toscrape.com": {"concurrency": 16, "delay": 0.1}, - "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, + "example.com": {"concurrency": 16, "delay": 0.1}, } Additional keys like ``"jitter"`` and ``"backoff"`` can be used here and are diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index a28333c90..6a3c155d5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -162,9 +162,6 @@ class _DeprecatedSlotView: def __repr__(self) -> str: return f"_DeprecatedSlotView({self._key!r})" - def __str__(self) -> str: - return f"_DeprecatedSlotView({self._key!r})" - class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): """Deprecated mapping view of active downloads, keyed by slot name.""" diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 54b429fb7..c2d57b750 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -1,12 +1,11 @@ from __future__ import annotations -import datetime as dt import logging -from email.utils import parsedate_to_datetime from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured from scrapy.throttling import _load_exceptions, iter_scopes +from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: @@ -22,47 +21,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -def _decoded_header(response: Response, name: str) -> str | None: - """Return the stripped UTF-8 value of the *name* header of *response*, or - ``None`` if it is absent or not valid UTF-8.""" - raw = response.headers.get(name) - if not raw: - return None - try: - return raw.decode("utf-8").strip() - except UnicodeDecodeError: - return None - - -def _parse_retry_after(response: Response) -> float | None: - value = _decoded_header(response, "Retry-After") - if value is None: - return None - if value.isdigit(): - return float(value) # seconds - try: - date = parsedate_to_datetime(value) - except (TypeError, ValueError, OverflowError): - return None - if date.tzinfo is None: - date = date.replace(tzinfo=dt.timezone.utc) - now = dt.datetime.now(dt.timezone.utc) - seconds_to_wait = (date - now).total_seconds() - # Keep sub-second precision (a date less than a second away must not be - # truncated to 0 and dropped); a past or present date yields no delay. - return max(0.0, seconds_to_wait) or None - - -def _parse_ratelimit_reset(response: Response) -> float | None: - value = _decoded_header(response, "RateLimit-Reset") - if value is None: - return None - try: - return float(value) - except ValueError: - return None - - class BackoffMiddleware: """Downloader middleware that drives :ref:`backoff ` from download outcomes. diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 40fa5511c..35b347d08 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -7,10 +7,10 @@ from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string from scrapy import signals -from scrapy.downloadermiddlewares.backoff import _parse_retry_after from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import HtmlResponse, Response from scrapy.spidermiddlewares.referer import RefererMiddleware +from scrapy.utils._headers import _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import global_object_name diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 480bb3a8f..2681c3983 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -1277,8 +1277,7 @@ class ThrottlingScopeManager: if self._backoff_level == 0 or self._last_backoff_time is None: return if self._window <= 0: - # A non-positive window has no recovery cadence to step through (and - # would spin forever on a zero-length step), so recover at once. + # No window: no recovery cadence to step (would spin); recover at once. self._backoff_level = 0 self._delay = self._base_delay self._in_backoff_until = None @@ -1300,8 +1299,7 @@ class ThrottlingScopeManager: if not self._rampup_enabled: return if self._window <= 0: - # No window means no cadence to ramp up on (and a zero-length step - # would spin forever). + # No window: no rampup cadence to step (would spin). return if self._rampup_window_start is None: self._rampup_window_start = now @@ -1338,8 +1336,7 @@ class ThrottlingScopeManager: if self._quota is None: return if self._quota_window <= 0: - # A non-positive window has no reset cadence to step through (and - # would spin forever on a zero-length step), so keep it reset. + # No window: no reset cadence to step (would spin); keep it reset. self._consumed = 0.0 self._quota_window_start = now return From b26befe4c618051b46ad1552ac899e7559c269e2 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 07:49:27 +0200 Subject: [PATCH 47/55] Add missing files --- docs/intro/tutorial.rst | 3 +-- scrapy/utils/_headers.py | 49 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 scrapy/utils/_headers.py diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index d61b373ba..94e27386c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -150,8 +150,7 @@ delay for that domain only: "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, } -The polite defaults still apply to every other website. Remove this entry once -you move on to crawling your own sites. +The polite defaults still apply to every other website. How to run our spider --------------------- diff --git a/scrapy/utils/_headers.py b/scrapy/utils/_headers.py new file mode 100644 index 000000000..21352858c --- /dev/null +++ b/scrapy/utils/_headers.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import datetime as dt +from email.utils import parsedate_to_datetime +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scrapy.http import Response + + +def _decoded_header(response: Response, name: str) -> str | None: + """Return the stripped UTF-8 value of the *name* header of *response*, or + ``None`` if it is absent or not valid UTF-8.""" + raw = response.headers.get(name) + if not raw: + return None + try: + return raw.decode("utf-8").strip() + except UnicodeDecodeError: + return None + + +def _parse_retry_after(response: Response) -> float | None: + value = _decoded_header(response, "Retry-After") + if value is None: + return None + if value.isdigit(): + return float(value) # seconds + try: + date = parsedate_to_datetime(value) + except (TypeError, ValueError, OverflowError): + return None + if date.tzinfo is None: + date = date.replace(tzinfo=dt.timezone.utc) + now = dt.datetime.now(dt.timezone.utc) + seconds_to_wait = (date - now).total_seconds() + # Keep sub-second precision (a date less than a second away must not be + # truncated to 0 and dropped); a past or present date yields no delay. + return max(0.0, seconds_to_wait) or None + + +def _parse_ratelimit_reset(response: Response) -> float | None: + value = _decoded_header(response, "RateLimit-Reset") + if value is None: + return None + try: + return float(value) + except ValueError: + return None From 9da2b1cbc41e18951a0b6513128556ae43dd7743 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 07:54:54 +0200 Subject: [PATCH 48/55] Undo addon switch to a package --- scrapy/{addons/__init__.py => addons.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename scrapy/{addons/__init__.py => addons.py} (100%) diff --git a/scrapy/addons/__init__.py b/scrapy/addons.py similarity index 100% rename from scrapy/addons/__init__.py rename to scrapy/addons.py From 9fe2a229b63fb4d94fae6659d11a8a2f529d46af Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 10:24:00 +0200 Subject: [PATCH 49/55] Human review --- scrapy/core/scheduler.py | 2 +- scrapy/crawler.py | 6 +----- scrapy/downloadermiddlewares/backoff.py | 24 ++++++++---------------- scrapy/extensions/throttle.py | 6 ------ scrapy/pqueues.py | 16 ++++++---------- scrapy/throttling.py | 12 ++---------- scrapy/utils/misc.py | 5 +++++ 7 files changed, 23 insertions(+), 48 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d863ab20c..78d13bcb4 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -516,7 +516,7 @@ def _queue_supports_peek(queue_cls: type) -> bool: class ThrottlingAwareScheduler(Scheduler): - """A :class:`Scheduler` that only ever hands the engine requests whose + """A :setting:`SCHEDULER` that only ever hands the engine requests whose :ref:`throttling scopes ` allow them to be sent **right now**. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index e5df6f15f..0198ef93e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -105,10 +105,6 @@ class Crawler: self.addons.load_settings(self.settings) self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") - # 'max_concurrent_requests' historically overrode the per-domain slot - # concurrency, which is now THROTTLING_SCOPE_CONCURRENCY (see - # scrapy.throttling._default_scope_concurrency); the old deprecation - # message pointed at CONCURRENT_REQUESTS, which never matched that. self._apply_deprecated_spider_attr( "max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY" ) @@ -189,7 +185,7 @@ class Crawler: return warnings.warn( f"The {attr!r} spider attribute is deprecated. Use the {setting} " - f"setting or per-domain THROTTLING_SCOPES instead.", + f"setting or THROTTLING_SCOPES instead.", category=ScrapyDeprecationWarning, stacklevel=3, ) diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index c2d57b750..4925973ed 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -4,9 +4,10 @@ import logging from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured -from scrapy.throttling import _load_exceptions, iter_scopes +from scrapy.throttling import iter_scopes from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg +from scrapy.utils.misc import _load_objects if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -37,25 +38,20 @@ class BackoffMiddleware: See :ref:`throttling` for details. """ + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + def __init__(self, crawler: Crawler): if not crawler.settings.getbool("BACKOFF_ENABLED"): raise NotConfigured - # Throttling is a core, always-on subsystem: THROTTLING_MANAGER has a - # non-None default and is instantiated before the downloader is built, - # so crawler.throttler is always set here (the engine likewise asserts - # it in its download path). assert crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = crawler.throttler settings = crawler.settings - # Union of the global backoff triggers and every per-scope override: a - # response status (or exception type) outside it cannot trigger backoff - # for any scope, so the scopes of such a request need not be resolved. - # Each scope still makes the final decision via its scope manager's - # triggers_backoff_* methods (which read the per-scope overrides). self._http_codes: set[int] = { int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") } - self._exceptions: tuple[type[BaseException], ...] = _load_exceptions( + self._exceptions: tuple[type[BaseException], ...] = _load_objects( settings.getlist("BACKOFF_EXCEPTIONS") ) for scope_config in settings.getdict("THROTTLING_SCOPES").values(): @@ -63,11 +59,7 @@ class BackoffMiddleware: if "http_codes" in backoff: self._http_codes.update(int(code) for code in backoff["http_codes"]) if "exceptions" in backoff: - self._exceptions += _load_exceptions(backoff["exceptions"]) - - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler) + self._exceptions += _load_objects(backoff["exceptions"]) @_warn_spider_arg def process_response( diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 5a5bd540f..93f82268f 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -44,7 +44,6 @@ class AutoThrottle: f"AUTOTHROTTLE_TARGET_CONCURRENCY " f"({self.target_concurrency!r}) must be higher than 0." ) - # Scopes whose start delay has already been applied (see _scope_delay). self._started_scopes: set[str] = set() crawler.signals.connect(self._spider_opened, signal=signals.spider_opened) crawler.signals.connect( @@ -81,11 +80,6 @@ class AutoThrottle: ): return - # AutoThrottle predates throttling scopes, so it adjusts the delay of - # the request's domain scope, matching its historical per-domain slots. - # Key by hostname (not netloc) to match the default ThrottlingManager - # scope, so the adjusted delay applies to the scope actually enforced - # even for non-default ports. scope_id = urlparse_cached(request).hostname or "" olddelay = self._scope_delay(throttler, scope_id) newdelay = self._adjust_delay(olddelay, latency, response) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 69a0e7ea9..25008cc17 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -237,10 +237,6 @@ class ScrapyPriorityQueue: """ if self.curprio is None: return None - # Mirror pop(): at a given priority it drains the regular queue before - # the start queue, so peek() must report the regular head first for the - # two to agree on "the next request" (a throttling-aware queue relies on - # this to check readiness of the exact request pop() will return). try: queue = self.queues[self.curprio] except KeyError: @@ -509,18 +505,18 @@ class ThrottlingAwarePriorityQueue: self.key: str = key self.crawler: Crawler = crawler - # scope set -> priority queue self.pqueues: dict[frozenset[ScopeID], ScrapyPriorityQueue] = {} - # Min-heap of (deadline, seq, scope_set, request) for requests held back - # by a per-request throttling_delay; seq keeps ordering stable and - # avoids comparing requests when deadlines tie. - self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] - self._delayed_seq: int = 0 if slot_startprios: for set_key, startprios in slot_startprios.items(): scope_set = _scope_set_from_key(set_key) self.pqueues[scope_set] = self.pqfactory(scope_set, startprios) + # Min-heap of (deadline, seq, scope_set, request) for requests held + # back by a per-request throttling_delay; seq keeps ordering stable and + # avoids comparing requests when deadlines tie. + self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] + self._delayed_seq: int = 0 + def pqfactory( self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = () ) -> ScrapyPriorityQueue: diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 2681c3983..5c77dad6c 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -19,7 +19,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.misc import _load_objects, build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -126,14 +126,6 @@ def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | yield scope, None -def _load_exceptions(exceptions: Iterable[Any]) -> tuple[type[BaseException], ...]: - """Resolve *exceptions* (exception classes or their import paths) to a tuple - of exception classes.""" - return tuple( - load_object(exc) if isinstance(exc, str) else exc for exc in exceptions - ) - - def _effective_priority(settings: BaseSettings, name: str) -> int: """Return the priority of setting *name*, treating an unset setting (no priority, ``None``) as just below ``"default"`` so it never wins over one @@ -1180,7 +1172,7 @@ class ThrottlingScopeManager: "http_codes", settings.getlist("BACKOFF_HTTP_CODES") ) } - self._backoff_exceptions: tuple[type[BaseException], ...] = _load_exceptions( + self._backoff_exceptions: tuple[type[BaseException], ...] = _load_objects( backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) ) self._window: float = settings.getfloat("BACKOFF_WINDOW") diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 20e7cb381..57b526be6 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any: return obj +def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]: + """Resolve *objects* (objects or import paths) to a tuple of objects.""" + return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects) + + def walk_modules_iter(path: str) -> Iterable[ModuleType]: """Loads a module and all its submodules from the given module path and returns them. If *any* module throws an exception while importing, that From f933d733230e811f9572f270f6e495afff3bade1 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 11:15:26 +0200 Subject: [PATCH 50/55] WIP --- docs/topics/throttling.rst | 21 +--- scrapy/core/downloader/handlers/_base_http.py | 25 ++++ .../downloader/handlers/_base_streaming.py | 1 + scrapy/core/downloader/handlers/http11.py | 18 +-- scrapy/core/engine.py | 119 +++++++++--------- scrapy/settings/default_settings.py | 3 - scrapy/throttling.py | 31 +++++ tests/test_engine.py | 83 ++++++------ 8 files changed, 161 insertions(+), 140 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index bfa126fc8..b231e8586 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -617,8 +617,8 @@ By default, throttling is enforced at the engine, where a request waiting on its :ref:`throttling scopes ` holds a concurrency slot. In a crawl that mixes heavily-throttled scopes with unthrottled ones, this can let throttled requests starve unthrottled ones that could be sent right away -(**head-of-line blocking**; Scrapy logs a warning, see -:setting:`DELAYED_REQUESTS_WARN_THRESHOLD`). +(**head-of-line blocking**; Scrapy logs a warning the first time throttled +requests start consuming the global concurrency budget while they wait). :class:`~scrapy.core.scheduler.ThrottlingAwareScheduler` avoids this. To enable it: @@ -992,23 +992,6 @@ Additional settings by one :setting:`BACKOFF_DELAY_FACTOR` step toward the configured value. A new trigger resets the countdown. -- .. setting:: DELAYED_REQUESTS_WARN_THRESHOLD - - :setting:`DELAYED_REQUESTS_WARN_THRESHOLD` (default: ``500``) - - Number of requests held back by :ref:`throttling ` at which - Scrapy logs a warning, to help detect throttling configurations that hold - back more requests than expected. - - While throttled, requests in the :ref:`scheduler ` - remain in the scheduler. However, requests sent with :meth:`engine.download() - ` bypass the scheduler, - including requests sent by some built-in :ref:`components - ` and :ref:`inline requests `. When - such requests are throttled, they are paused and kept in memory, along - with any run time context from the code that is sending them. If they - accumulate, they can become a memory issue. - - .. setting:: RANDOMIZE_DOWNLOAD_DELAY :setting:`RANDOMIZE_DOWNLOAD_DELAY` (default: ``True``) diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py index 83d130462..2ca8ff353 100644 --- a/scrapy/core/downloader/handlers/_base_http.py +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -7,11 +7,36 @@ from .base import BaseDownloadHandler if TYPE_CHECKING: from scrapy.crawler import Crawler + from scrapy.settings import BaseSettings class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): """Base class for built-in HTTP download handlers.""" + @staticmethod + def _max_per_host_concurrency(settings: BaseSettings) -> int: + """Highest per-host concurrency the throttler may admit: the per-domain + limit, the default ``other``-scope limit, and any explicit + :setting:`THROTTLING_SCOPES` concurrency. + + A scope with :ref:`rampup ` enabled has no configured + concurrency ceiling; it grows toward :setting:`CONCURRENT_REQUESTS`, so + it counts as that. And since :setting:`CONCURRENT_REQUESTS` caps the + total number of requests in flight, no host can ever exceed it, so it is + also the upper bound of the result. + """ + global_concurrency = settings.getint("CONCURRENT_REQUESTS") + candidates = [ + settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), + settings.getint("THROTTLING_SCOPE_CONCURRENCY"), + ] + for scope in settings.getdict("THROTTLING_SCOPES").values(): + if scope.get("rampup"): + candidates.append(global_concurrency) + elif "concurrency" in scope: + candidates.append(int(scope["concurrency"])) + return min(max(candidates), global_concurrency) + def __init__(self, crawler: Crawler): super().__init__(crawler) self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py index 6742781e6..d74a7d120 100644 --- a/scrapy/core/downloader/handlers/_base_streaming.py +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -85,6 +85,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon ) self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") + self._pool_size_per_host: int = self._max_per_host_concurrency(crawler.settings) @staticmethod @abstractmethod diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 49422b0bf..cb46f2103 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -92,22 +92,8 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): from twisted.internet import reactor self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True) - # Keep enough persistent connections per host to match the highest - # per-host concurrency the throttler may admit: the per-domain limit, - # the default "other"-scope limit, and any explicit THROTTLING_SCOPES - # concurrency. Per-scope concurrency can grow beyond this at runtime - # (rampup); the excess simply uses non-persistent connections. - scope_concurrencies = [ - scope["concurrency"] - for scope in crawler.settings.getdict("THROTTLING_SCOPES").values() - if "concurrency" in scope - ] - self._pool.maxPersistentPerHost = max( - [ - crawler.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), - crawler.settings.getint("THROTTLING_SCOPE_CONCURRENCY"), - *scope_concurrencies, - ] + self._pool.maxPersistentPerHost = self._max_per_host_concurrency( + crawler.settings ) self._pool._factory.noisy = False diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2469866b9..217d52826 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -138,14 +138,18 @@ class ExecutionEngine: # ``_enqueue_request_async``), so the spider is not considered idle # while a request is still on its way into the scheduler. self._scheduling: int = 0 - # A coalesced wakeup timer, armed when a throttling-aware scheduler - # reports that all pending requests are time-blocked (see - # ``get_next_request_delay``). - self._throttling_wakeup: CallLaterResult | None = None - self._delayed_requests_warn_threshold: int = self.settings.getint( - "DELAYED_REQUESTS_WARN_THRESHOLD" - ) - self._delayed_requests_warned: bool = False + # A coalesced wakeup timer, armed when the scheduler reports (through + # ``get_next_request_delay``) that every pending request is time-blocked. + self._delay_wakeup: CallLaterResult | None = None + # The scheduler's ``get_next_request_delay`` bound method, cached once + # per crawl in ``open_spider_async`` (``None`` if the scheduler does not + # expose one). Used both to arm the wakeup timer and to tell whether the + # scheduler holds time-blocked requests itself. + self._get_next_request_delay: Callable[[], float | None] | None = None + # Whether the scheduler exposes enqueue_request_async, cached once per + # crawl in ``open_spider_async``; checked for every scheduled request. + self._scheduler_enqueues_async: bool = False + self._throttling_backout_warned: bool = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -279,14 +283,14 @@ class ExecutionEngine: def pause(self) -> None: self.paused = True - self._cancel_throttling_wakeup() + self._cancel_delay_wakeup() def unpause(self) -> None: self.paused = False - # pause() cancels the coalesced throttling wakeup and the loop stops + # pause() cancels the coalesced delay wakeup and the loop stops # re-running itself while paused, so re-run it here: this re-arms the - # wakeup and keeps a crawl held only by a time-based throttling gate - # (nothing in flight to re-run the loop from _download) from stalling. + # wakeup and keeps a crawl held only by a time-based gate (nothing in + # flight to re-run the loop from _download) from stalling. if self._slot is not None: self._slot.nextcall.schedule() @@ -355,25 +359,25 @@ class ExecutionEngine: if self._slot is None or self._slot.closing is not None or self.paused: return - self._cancel_throttling_wakeup() + self._cancel_delay_wakeup() while not self.needs_backout(): if not self._start_scheduled_request(): break - self._maybe_arm_throttling_wakeup() + self._maybe_arm_delay_wakeup() if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() - def _cancel_throttling_wakeup(self) -> None: - if self._throttling_wakeup is not None: - self._throttling_wakeup.cancel() - self._throttling_wakeup = None + def _cancel_delay_wakeup(self) -> None: + if self._delay_wakeup is not None: + self._delay_wakeup.cancel() + self._delay_wakeup = None - def _maybe_arm_throttling_wakeup(self) -> None: - """Arm a single coalesced wakeup when the scheduler is throttling-aware - and reports that every pending request is time-blocked. + def _maybe_arm_delay_wakeup(self) -> None: + """Arm a single coalesced wakeup when the scheduler reports that every + pending request is time-blocked. Concurrency-blocked states do not need this: a freed slot already re-runs the loop from :meth:`_download`'s ``finally``. Only time-based @@ -381,21 +385,19 @@ class ExecutionEngine: would re-run the loop while they are closed. """ assert self._slot is not None - scheduler = self._slot.scheduler - delay_fn = getattr(scheduler, "get_next_request_delay", None) - if delay_fn is None or not scheduler.has_pending_requests(): + if ( + self._get_next_request_delay is None + or not self._slot.scheduler.has_pending_requests() + ): return - delay = delay_fn() - # ``delay`` is ``None`` when no pending request is time-blocked, and - # ``0`` when some request is ready right now but could not be sent (e.g. - # the downloader is at capacity). Neither case needs a timer: a freed - # slot already re-runs the loop from :meth:`_download`'s ``finally``, - # and arming a ``0``-second timer here would busy-loop the engine while - # the downloader stays full. Only a positive delay, i.e. a time-based - # gate that nothing else would wake us for, needs one. + # A positive delay means a time-based gate nothing else would wake us + # for. ``None`` (nothing time-blocked) and ``0`` (something is ready but + # could not be sent, e.g. the downloader is full) are handled elsewhere: + # a freed slot re-runs the loop, and a ``0``-second timer would busy-loop. + delay = self._get_next_request_delay() if delay is None or delay <= 0: return - self._throttling_wakeup = call_later(delay, self._slot.nextcall.schedule) + self._delay_wakeup = call_later(delay, self._slot.nextcall.schedule) def needs_backout(self) -> bool: """Returns ``True`` if no more requests can be sent at the moment, or @@ -423,29 +425,28 @@ class ExecutionEngine: """ if not self._throttling_waiting: return False - return ( + if ( len(self._throttling_waiting) + len(self.downloader.active) - >= self.downloader.total_concurrency - ) + < self.downloader.total_concurrency + ): + return False + self._maybe_warn_throttling_backout() + return True - def _maybe_warn_delayed_requests(self) -> None: - if self._delayed_requests_warned: + def _maybe_warn_throttling_backout(self) -> None: + if self._throttling_backout_warned: return - if len(self._throttling_waiting) < self._delayed_requests_warn_threshold: + # A throttling-aware scheduler holds time-blocked requests itself instead + # of letting them pile up in _throttling_waiting and back-pressure the + # engine here, so the warning does not apply when one is in use. + if self._get_next_request_delay is not None: return - self._delayed_requests_warned = True - recommendation = "" - # A throttling-aware scheduler holds throttled requests in the - # scheduler instead of in _throttling_waiting, so it does not hit this - # path; recommend it only when it is not already in use. - scheduler = self._slot.scheduler if self._slot is not None else None - if scheduler is None or not hasattr(scheduler, "get_next_request_delay"): - recommendation = ( - " Consider switching to scrapy.core.scheduler.ThrottlingAwareScheduler." - ) + self._throttling_backout_warned = True logger.warning( - f"There are {len(self._throttling_waiting)} requests held back by " - f"throttling. See DELAYED_REQUESTS_WARN_THRESHOLD.{recommendation}", + "Throttling is holding requests back and they are now consuming the " + "global concurrency budget while they wait for a free slot. Consider " + "switching to scrapy.core.scheduler.ThrottlingAwareScheduler, which " + "holds throttled requests without consuming concurrency slots.", extra={"spider": self.spider}, ) @@ -549,7 +550,7 @@ class ExecutionEngine: if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest): return scheduler = self._slot.scheduler - if hasattr(scheduler, "enqueue_request_async"): + if self._scheduler_enqueues_async: self._scheduling += 1 _schedule_coro(self._enqueue_request_async(request)) return @@ -559,10 +560,9 @@ class ExecutionEngine: ) async def _enqueue_request_async(self, request: Request) -> None: - # The counter is incremented in _schedule_request before this coroutine - # is scheduled, so it must be decremented here even on an early exit, - # otherwise spider_is_idle() never reports idle. The slot can be torn - # down (spider stopping) between the increment and this running. + # _scheduling is incremented in _schedule_request before this coroutine + # is scheduled, so it must be decremented on every path (hence finally), + # otherwise spider_is_idle() never reports idle. try: if self._slot is None: return @@ -618,7 +618,6 @@ class ExecutionEngine: """Wait at the throttling gate before *request* is sent, tracking it as held meanwhile.""" self._throttling_waiting.add(request) - self._maybe_warn_delayed_requests() throttler = self.crawler.throttler assert throttler is not None try: @@ -689,6 +688,10 @@ class ExecutionEngine: self.spider = self.crawler.spider nextcall = CallLaterOnce(self._start_scheduled_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) + self._get_next_request_delay = getattr( + scheduler, "get_next_request_delay", None + ) + self._scheduler_enqueues_async = hasattr(scheduler, "enqueue_request_async") self._slot = _Slot(close_if_idle, nextcall, scheduler) self._start = await self.scraper.spidermw.process_start() if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): @@ -765,7 +768,7 @@ class ExecutionEngine: "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - self._cancel_throttling_wakeup() + self._cancel_delay_wakeup() try: await self._slot.close() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b9b9d19f3..cc145d241 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -58,7 +58,6 @@ __all__ = [ "DEFAULT_DROPITEM_LOG_LEVEL", "DEFAULT_ITEM_CLASS", "DEFAULT_REQUEST_HEADERS", - "DELAYED_REQUESTS_WARN_THRESHOLD", "DEPTH_LIMIT", "DEPTH_PRIORITY", "DEPTH_STATS_VERBOSE", @@ -300,8 +299,6 @@ DEFAULT_REQUEST_HEADERS = { "Accept-Language": "en", } -DELAYED_REQUESTS_WARN_THRESHOLD = 500 - DEPTH_LIMIT = 0 DEPTH_PRIORITY = 0 DEPTH_STATS_VERBOSE = False diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 5c77dad6c..646506f47 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -207,6 +207,36 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: ) +def _warn_on_unachievable_concurrency(settings: BaseSettings) -> None: + """Warn about configured concurrency limits that exceed + :setting:`CONCURRENT_REQUESTS`. Call once per crawl (see + :meth:`ThrottlingManager.__init__`). + + :setting:`CONCURRENT_REQUESTS` caps the total number of requests in flight, + so a per-scope (or per-domain) concurrency limit above it can never be + reached. Rampup is not flagged: it has no configured ceiling and simply + grows toward :setting:`CONCURRENT_REQUESTS`. + """ + global_concurrency = settings.getint("CONCURRENT_REQUESTS") + offenders: list[str] = [ + f"{name}={settings.getint(name)}" + for name in ("CONCURRENT_REQUESTS_PER_DOMAIN", "THROTTLING_SCOPE_CONCURRENCY") + if settings.getint(name) > global_concurrency + ] + offenders += [ + f"THROTTLING_SCOPES[{scope_id!r}]['concurrency']={config['concurrency']}" + for scope_id, config in settings.getdict("THROTTLING_SCOPES").items() + if config.get("concurrency") is not None + and int(config["concurrency"]) > global_concurrency + ] + if offenders: + logger.warning( + f"The following concurrency settings exceed CONCURRENT_REQUESTS " + f"({global_concurrency}), which caps the total number of requests in " + f"flight, so they cannot be reached: {', '.join(offenders)}." + ) + + def _to_scope_dict(scopes: RequestScopes) -> dict[ScopeID, float | None]: """Normalize *scopes* (``None``, a scope id, an iterable of scope ids or a ``{scope_id: quota}`` dict) into a ``{scope_id: quota}`` dict, using ``None`` @@ -505,6 +535,7 @@ class ThrottlingManager: def __init__(self, crawler: Crawler) -> None: self.crawler = crawler _warn_on_deprecated_concurrency(crawler.settings) + _warn_on_unachievable_concurrency(crawler.settings) self._debug = crawler.settings.getbool("THROTTLING_DEBUG") self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") self._robotstxt_obey = crawler.settings.getbool( diff --git a/tests/test_engine.py b/tests/test_engine.py index a7f81e19d..4bca525c5 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -663,74 +663,69 @@ class TestEngineThrottling: yield engine engine.downloader.close() - def test_pause_cancels_throttling_wakeup(self, engine): + def test_pause_cancels_delay_wakeup(self, engine): wakeup = Mock() - engine._throttling_wakeup = wakeup + engine._delay_wakeup = wakeup engine.pause() assert engine.paused is True wakeup.cancel.assert_called_once_with() - assert engine._throttling_wakeup is None + assert engine._delay_wakeup is None engine.unpause() assert engine.paused is False @pytest.mark.requires_reactor # call_later() needs a reactor or asyncio loop - def test_maybe_arm_throttling_wakeup_arms_timer(self, engine): - scheduler = Mock() - scheduler.has_pending_requests.return_value = True - scheduler.get_next_request_delay.return_value = 5.0 + def test_maybe_arm_delay_wakeup_arms_timer(self, engine): + engine._get_next_request_delay = lambda: 5.0 engine._slot = Mock() - engine._slot.scheduler = scheduler - engine._maybe_arm_throttling_wakeup() - assert engine._throttling_wakeup is not None + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is not None # Cancel the scheduled reactor call so it does not leak into other tests. - engine._cancel_throttling_wakeup() + engine._cancel_delay_wakeup() - def test_maybe_arm_throttling_wakeup_no_delay(self, engine): - scheduler = Mock() - scheduler.has_pending_requests.return_value = True - scheduler.get_next_request_delay.return_value = None + def test_maybe_arm_delay_wakeup_no_delay(self, engine): + engine._get_next_request_delay = lambda: None engine._slot = Mock() - engine._slot.scheduler = scheduler - engine._maybe_arm_throttling_wakeup() - assert engine._throttling_wakeup is None + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is None - def test_maybe_arm_throttling_wakeup_zero_delay(self, engine): + def test_maybe_arm_delay_wakeup_zero_delay(self, engine): # A 0 delay means a request is ready but could not be sent (e.g. the # downloader is at capacity); arming a 0-second timer would busy-loop # the engine, so no timer must be armed. - scheduler = Mock() - scheduler.has_pending_requests.return_value = True - scheduler.get_next_request_delay.return_value = 0.0 + engine._get_next_request_delay = lambda: 0.0 engine._slot = Mock() - engine._slot.scheduler = scheduler - engine._maybe_arm_throttling_wakeup() - assert engine._throttling_wakeup is None + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is None - def test_warn_delayed_requests(self, engine): - engine._delayed_requests_warn_threshold = 1 - engine._throttling_waiting = {Request("http://a.example")} + def test_maybe_arm_delay_wakeup_not_supported(self, engine): + # A scheduler without get_next_request_delay never arms a timer. + engine._get_next_request_delay = None engine._slot = Mock() - # A scheduler without get_next_request_delay is not throttling-aware, so the - # warning recommends switching to one. - engine._slot.scheduler = Mock(spec=BaseScheduler) + engine._slot.scheduler.has_pending_requests.return_value = True + engine._maybe_arm_delay_wakeup() + assert engine._delay_wakeup is None + + def test_maybe_warn_throttling_backout(self, engine): + # A scheduler without get_next_request_delay is not throttling-aware, so + # the warning recommends switching to one. + engine._get_next_request_delay = None with LogCapture() as log: - engine._maybe_warn_delayed_requests() + engine._maybe_warn_throttling_backout() # A second call is a no-op (the warning is emitted only once). - engine._maybe_warn_delayed_requests() - assert engine._delayed_requests_warned is True - log_text = str(log) - assert "requests held back by throttling" in log_text - assert log_text.count("ThrottlingAwareScheduler") == 1 + engine._maybe_warn_throttling_backout() + assert engine._throttling_backout_warned is True + assert str(log).count("ThrottlingAwareScheduler") == 1 - def test_warn_delayed_requests_throttling_aware(self, engine): - engine._delayed_requests_warn_threshold = 1 - engine._throttling_waiting = {Request("http://a.example")} - engine._slot = Mock() + def test_maybe_warn_throttling_backout_throttling_aware(self, engine): # A throttling-aware scheduler (one with get_next_request_delay) holds - # throttled requests itself, so no switch is recommended. - engine._slot.scheduler = Mock() + # throttled requests itself, so no warning is emitted. + engine._get_next_request_delay = lambda: None with LogCapture() as log: - engine._maybe_warn_delayed_requests() + engine._maybe_warn_throttling_backout() + assert engine._throttling_backout_warned is False assert "ThrottlingAwareScheduler" not in str(log) def test_spider_is_idle_false_while_scheduling(self, engine): From 1fb8b09334828431c3ee97a6b49fecdf3ae4efa9 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 12:07:34 +0200 Subject: [PATCH 51/55] Remove rampup --- docs/topics/throttling.rst | 82 +++----------- scrapy/core/downloader/handlers/_base_http.py | 18 ++- scrapy/settings/default_settings.py | 3 - scrapy/throttling.py | 104 ++---------------- tests/test_throttling.py | 60 ---------- 5 files changed, 28 insertions(+), 239 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index b231e8586..d087ab302 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -166,9 +166,14 @@ until it is back to the configured value. A new trigger resets the countdown. This exponential-increase / linear-decrease pattern, similar to TCP congestion control, makes a scope back off quickly when a server is unhappy and return to -full speed gradually once it recovers. To keep a scope hovering around a target -rate instead of repeatedly probing and backing off, enable :ref:`rampup -`. +full speed gradually once it recovers. + +Backoff only ever *tightens* a scope, and recovery never goes past the +configured value: the delay can grow above the configured ``"delay"`` and then +recover back down to it, but never below it, and backoff never raises the +concurrency limit. So set the ``"delay"`` and ``"concurrency"`` you actually +want for a scope; backoff makes things gentler from there when a server pushes +back, and returns to those values once it recovers. Backoff triggers are detected by the :class:`~scrapy.downloadermiddlewares.backoff.BackoffMiddleware`, a built-in @@ -211,61 +216,6 @@ it. So a scope can, for example, treat an extra status code as a backoff trigger, or stop treating one of the defaults as a trigger, independently of every other scope. -.. _rampup: - -Rampup -====== - -When using APIs that charge per request, like web scraping APIs, you often want -to maximize throughput while staying within rate limits. To do that, set -``"rampup"`` to ``True`` in :setting:`THROTTLING_SCOPES`: - -.. code-block:: python - :caption: ``settings.py`` - - THROTTLING_SCOPES = { - "api.toscrape.com": { - "rampup": True, - }, - } - -Rampup increases concurrency or lowers delay as needed based on the following -setting: - -- .. setting:: RAMPUP_BACKOFF_TARGET - - :setting:`RAMPUP_BACKOFF_TARGET` (default: ``1``) - - Target number of backoff responses per :setting:`BACKOFF_WINDOW` that - :ref:`rampup ` aims for when probing the rate limit of a scope. - -For every :setting:`BACKOFF_WINDOW` that stays **below** -:setting:`RAMPUP_BACKOFF_TARGET` backoff triggers, rampup increases throughput -one step: it first lowers the delay, and once the delay reaches its minimum it -raises the concurrency limit of the scope. Windows that reach or exceed the -target do not ramp up; the rate is -reduced by normal :ref:`backoff ` (which grows the delay) instead. -Rampup only ever probes upward, so the rate settles around the most throughput -a scope allows while triggering fewer than :setting:`RAMPUP_BACKOFF_TARGET` -rate-limit responses per window. - -Rampup behavior can be fine-tuned per scope by giving ``"rampup"`` a dict -instead of ``True``: - -.. code-block:: python - :caption: ``settings.py`` - - THROTTLING_SCOPES = { - "api.toscrape.com": { - "rampup": { - "backoff_target": 1, # overrides RAMPUP_BACKOFF_TARGET - "delay_factor": 0.5, # multiply the delay by this on each ramp-up step - "min_delay": 0.05, # do not ramp the delay below this - }, - }, - } - - .. _retry-after: .. _rate-limiting-headers: @@ -485,9 +435,6 @@ following keys: ``window`` (:class:`float`) Quota window in seconds. Defaults to :setting:`THROTTLING_WINDOW`. -``rampup`` (:class:`bool` or :class:`dict`) - Enables :ref:`rampup ` for the scope. - ``backoff`` (:class:`~scrapy.throttling.BackoffConfig`) Per-scope :ref:`backoff overrides `. @@ -572,11 +519,10 @@ as a string): THROTTLING_SCOPE_MANAGER = "myproject.throttling.MyThrottlingScopeManager" For each throttling scope, an instance of this class is created to manage any -gradual :ref:`backoff ` or :ref:`rampup ` required at run -time. +gradual :ref:`backoff ` required at run time. You can implement your own throttling scope manager if you wish to change the -backoff or rampup behavior beyond what settings allow. +backoff behavior beyond what settings allow. You can also define a custom throttling scope manager for a specific throttling scope by setting the ``"manager"`` key in the :setting:`THROTTLING_SCOPES` @@ -984,9 +930,9 @@ Additional settings :setting:`BACKOFF_WINDOW` (default: ``60.0``) - Time window, in seconds, used by :ref:`backoff ` and - :ref:`rampup `. During backoff, a :ref:`throttling scope - ` must go this many seconds without a new backoff + Time window, in seconds, used by :ref:`backoff `. A + :ref:`throttling scope ` must go this many seconds + without a new backoff trigger (an HTTP error code from :setting:`BACKOFF_HTTP_CODES` or an exception from :setting:`BACKOFF_EXCEPTIONS`) before its delay decreases by one :setting:`BACKOFF_DELAY_FACTOR` step toward the configured value. @@ -1062,8 +1008,6 @@ API .. autoclass:: scrapy.throttling.BackoffConfig -.. autoclass:: scrapy.throttling.RampupConfig - .. autofunction:: scrapy.throttling.scope_cache .. autofunction:: scrapy.throttling.add_scope .. autofunction:: scrapy.throttling.iter_scopes diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py index 2ca8ff353..66308389a 100644 --- a/scrapy/core/downloader/handlers/_base_http.py +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -19,22 +19,20 @@ class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): limit, the default ``other``-scope limit, and any explicit :setting:`THROTTLING_SCOPES` concurrency. - A scope with :ref:`rampup ` enabled has no configured - concurrency ceiling; it grows toward :setting:`CONCURRENT_REQUESTS`, so - it counts as that. And since :setting:`CONCURRENT_REQUESTS` caps the - total number of requests in flight, no host can ever exceed it, so it is - also the upper bound of the result. + Since :setting:`CONCURRENT_REQUESTS` caps the total number of requests in + flight, no host can ever exceed it, so it is also the upper bound of the + result. """ global_concurrency = settings.getint("CONCURRENT_REQUESTS") candidates = [ settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), settings.getint("THROTTLING_SCOPE_CONCURRENCY"), ] - for scope in settings.getdict("THROTTLING_SCOPES").values(): - if scope.get("rampup"): - candidates.append(global_concurrency) - elif "concurrency" in scope: - candidates.append(int(scope["concurrency"])) + candidates += [ + int(scope["concurrency"]) + for scope in settings.getdict("THROTTLING_SCOPES").values() + if "concurrency" in scope + ] return min(max(candidates), global_concurrency) def __init__(self, crawler: Crawler): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index cc145d241..20859ae30 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -174,7 +174,6 @@ __all__ = [ "PERIODIC_LOG_DELTA", "PERIODIC_LOG_STATS", "PERIODIC_LOG_TIMING_ENABLED", - "RAMPUP_BACKOFF_TARGET", "RANDOMIZE_DOWNLOAD_DELAY", "REACTOR_THREADPOOL_MAXSIZE", "REDIRECT_ENABLED", @@ -521,8 +520,6 @@ PERIODIC_LOG_DELTA = None PERIODIC_LOG_STATS = None PERIODIC_LOG_TIMING_ENABLED = False -RAMPUP_BACKOFF_TARGET = 1 - RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 diff --git a/scrapy/throttling.py b/scrapy/throttling.py index 646506f47..e460a5163 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -46,20 +46,6 @@ class BackoffConfig(TypedDict, total=False): jitter: float | list[float] -class RampupConfig(TypedDict, total=False): - """Per-scope override of the rampup settings. - - Used as the value of the ``"rampup"`` key of :class:`ThrottlingScopeConfig` - entries when fine-tuning rampup beyond a plain ``True``. Any key left out - falls back to its default (or, for ``backoff_target``, to - :setting:`RAMPUP_BACKOFF_TARGET`). - """ - - backoff_target: float - delay_factor: float - min_delay: float - - class ThrottlingScopeConfig(TypedDict, total=False): """Accepted keys of :setting:`THROTTLING_SCOPES` entries. @@ -78,7 +64,6 @@ class ThrottlingScopeConfig(TypedDict, total=False): quota: float window: float - rampup: bool | RampupConfig manager: str | type """Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for @@ -214,8 +199,7 @@ def _warn_on_unachievable_concurrency(settings: BaseSettings) -> None: :setting:`CONCURRENT_REQUESTS` caps the total number of requests in flight, so a per-scope (or per-domain) concurrency limit above it can never be - reached. Rampup is not flagged: it has no configured ceiling and simply - grows toward :setting:`CONCURRENT_REQUESTS`. + reached. """ global_concurrency = settings.getint("CONCURRENT_REQUESTS") offenders: list[str] = [ @@ -991,11 +975,6 @@ class ThrottlingScopeManagerProtocol(Protocol): "min_delay": 5.0, "jitter": [0.01, 0.33], }, - "rampup": { - "backoff_target": 1, - "delay_factor": 0.5, - "min_delay": 0.05, - }, } """ @@ -1128,8 +1107,8 @@ class ThrottlingScopeManager: """The default :setting:`THROTTLING_SCOPE_MANAGER` class. It implements a per-scope state machine covering delay, exponential - :ref:`backoff `, :ref:`rampup `, concurrency and - :ref:`quotas `: + :ref:`backoff `, concurrency and :ref:`quotas + `: - A base delay (the scope ``"delay"`` config, defaulting to :setting:`DOWNLOAD_DELAY`) is enforced between consecutive requests for @@ -1147,14 +1126,8 @@ class ThrottlingScopeManager: - After :setting:`BACKOFF_WINDOW` seconds without a new trigger, the delay recovers one step at a time back towards the base delay. - - When the scope is configured with a ``"concurrency"`` limit (or with - ``"rampup"``), no more than that many requests are allowed in flight at - once. - - - When the scope sets ``"rampup": True``, throughput is increased every - :setting:`BACKOFF_WINDOW` that stays under :setting:`RAMPUP_BACKOFF_TARGET` - backoff triggers, first by lowering the delay and then by raising the - concurrency limit. + - When the scope is configured with a ``"concurrency"`` limit, no more + than that many requests are allowed in flight at once. - When the scope is configured with a ``"quota"``, no more than that much quota is consumed per ``"window"`` (default: :setting:`THROTTLING_WINDOW`). @@ -1208,27 +1181,12 @@ class ThrottlingScopeManager: ) self._window: float = settings.getfloat("BACKOFF_WINDOW") - # Rampup. - rampup = config.get("rampup") - self._rampup_enabled: bool = bool(rampup) - rampup_config: dict[str, Any] = rampup if isinstance(rampup, dict) else {} - self._rampup_target: float = float( - rampup_config.get( - "backoff_target", settings.getfloat("RAMPUP_BACKOFF_TARGET") - ) - ) - self._rampup_delay_factor: float = float(rampup_config.get("delay_factor", 0.5)) - self._rampup_min_delay: float = float(rampup_config.get("min_delay", 0.0)) - # Concurrency. ``None`` means no scope-level limit (the downloader slots # enforce concurrency instead); a limit is only set when configured - # explicitly or implied by rampup. + # explicitly. configured_concurrency = config.get("concurrency") if configured_concurrency is not None: self._concurrency: int | None = int(configured_concurrency) - elif self._rampup_enabled: - # Rampup starts conservative at a single slot and probes upward. - self._concurrency = 1 else: self._concurrency = _default_scope_concurrency(settings) or None # Used as the load denominator when the scope enforces no explicit @@ -1253,8 +1211,6 @@ class ThrottlingScopeManager: self._slot_waiters: list[Deferred[None]] = [] self._consumed: float = 0.0 self._quota_window_start: float | None = None - self._rampup_window_start: float | None = None - self._rampup_backoffs: int = 0 @staticmethod def _now(now: float | None) -> float: @@ -1316,45 +1272,6 @@ class ThrottlingScopeManager: break self._delay = max(self._base_delay, self._delay / self._delay_factor) - def _maybe_rampup(self, now: float) -> None: - """Increase throughput once per :setting:`BACKOFF_WINDOW` that stays - under :setting:`RAMPUP_BACKOFF_TARGET` backoff triggers.""" - if not self._rampup_enabled: - return - if self._window <= 0: - # No window: no rampup cadence to step (would spin). - return - if self._rampup_window_start is None: - self._rampup_window_start = now - return - if now - self._rampup_window_start < self._window: - return - # Catch up with elapsed time but apply at most one ramp step per call: a - # scope that stayed idle for several windows must not ramp up - # cumulatively once it becomes active again (that would collapse the - # delay or jump the concurrency limit by several steps at once). - elapsed_windows = int((now - self._rampup_window_start) // self._window) - self._rampup_window_start += elapsed_windows * self._window - if self._rampup_backoffs < self._rampup_target: - self._rampup_step() - self._rampup_backoffs = 0 - - def _rampup_step(self) -> None: - # Backoff in progress: let it recover before probing again. - if self._backoff_level > 0: - return - if self._delay > self._rampup_min_delay: - # Lower only the effective delay while probing for headroom; the - # configured base delay is left untouched so it stays the recovery - # target on backoff and the value reported by get_base_delay(). - self._delay = max( - self._rampup_min_delay, self._delay * self._rampup_delay_factor - ) - else: - # Rampup is only enabled with a concurrency limit set. - assert self._concurrency is not None - self._concurrency += 1 - def _maybe_reset_quota(self, now: float) -> None: if self._quota is None: return @@ -1372,10 +1289,7 @@ class ThrottlingScopeManager: def can_send(self, now: float | None = None, amount: float | None = None) -> float: # can_send() only refreshes passive, time-based state (backoff recovery - # and the quota window) to reflect the current time; it performs no - # active throughput probing. That way a readiness check (is_ready() / - # get_time_until_ready()) has no side effect on the send rate: rampup - # only advances on an actual send, from record_sent(). + # and the quota window) to reflect the current time. now = self._now(now) self._recover(now) self._maybe_reset_quota(now) @@ -1402,9 +1316,6 @@ class ThrottlingScopeManager: self._last_seen = now if self._in_backoff_until is not None and now >= self._in_backoff_until: self._in_backoff_until = None - # An actual send is the cue to probe for more throughput (rampup), - # rather than a mere readiness check; see can_send(). - self._maybe_rampup(now) self._next_allowed_time = now + self._effective_delay() self._active += 1 if self._quota is not None and amount is not None: @@ -1457,7 +1368,6 @@ class ThrottlingScopeManager: self._last_seen = now self._last_backoff_time = now self._backoff_level += 1 - self._rampup_backoffs += 1 if delay is not None: # A hard delay (e.g. a Retry-After header) is a one-time gate: hold # the scope back for at least this long *once*, matching the HTTP diff --git a/tests/test_throttling.py b/tests/test_throttling.py index 3dacbe0ef..b05181a29 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttling.py @@ -638,16 +638,6 @@ class TestThrottlingScopeManager: assert scope.can_send(now=1.0, amount=5.0) == 0.0 assert scope._consumed == 0.0 - def test_zero_window_disables_rampup(self): - # A non-positive window must not make _maybe_rampup spin forever; rampup - # is simply disabled. - scope = _scope_manager( - settings={"BACKOFF_WINDOW": 0}, config={"id": "x", "rampup": True} - ) - scope.can_send(now=0.0) - scope.can_send(now=10_000.0) - assert scope._concurrency == scope._min_concurrency - def test_set_concurrency_fires_slot_event(self): scope = _scope_manager(config={"id": "x", "concurrency": 1}) scope.record_sent(now=0.0) @@ -711,42 +701,6 @@ class TestThrottlingScopeManager: scope.reconcile_quota(remaining=3.0, now=0.0) assert scope._consumed == pytest.approx(7.0) - def test_rampup_lowers_delay_when_quiet(self): - scope = _scope_manager( - {"BACKOFF_WINDOW": 10.0, "RANDOMIZE_DOWNLOAD_DELAY": False}, - { - "id": "x", - "delay": 4.0, - "rampup": {"delay_factor": 0.5, "min_delay": 0.5}, - }, - ) - scope.can_send(now=0.0) # start the rampup window - # A quiet window (no backoff) lowers the delay. - scope.can_send(now=10.0) - assert scope._delay == pytest.approx(2.0) - scope.can_send(now=20.0) - assert scope._delay == pytest.approx(1.0) - - def test_rampup_raises_concurrency_at_min_delay(self): - scope = _scope_manager( - {"BACKOFF_WINDOW": 10.0}, - {"id": "x", "delay": 0.0, "rampup": True, "min_concurrency": 1}, - ) - assert scope._concurrency == 1 - scope.can_send(now=0.0) - scope.can_send(now=10.0) - assert scope._concurrency == 2 - - def test_rampup_holds_when_target_met(self): - scope = _scope_manager( - {"BACKOFF_WINDOW": 10.0, "RAMPUP_BACKOFF_TARGET": 1}, - {"id": "x", "delay": 0.0, "rampup": True, "min_concurrency": 1}, - ) - scope.can_send(now=0.0) - scope.record_backoff(now=1.0) # one trigger == target -> hold, do not probe - scope.can_send(now=10.0) - assert scope._concurrency == 1 - class TestThrottlingManagerReadiness: """The synchronous readiness API used by a throttling-aware scheduler.""" @@ -1129,12 +1083,6 @@ class TestThrottlingScopeManagerEdges: # A randomized base delay lands within [0.5, 1.5] * delay. assert 1.0 <= scope.can_send(now=0.0) <= 3.0 - def test_rampup_target_as_range(self): - scope = _scope_manager( - config={"id": "x", "rampup": {"backoff_target": [1, 3]}}, - ) - assert scope._rampup_target == (1.0, 3.0) - def test_record_done_without_active(self): scope = _scope_manager(config={"id": "x"}) # Calling record_done() with nothing in flight is a harmless no-op. @@ -1160,14 +1108,6 @@ class TestThrottlingScopeManagerEdges: assert scope._base_delay == 0.5 assert scope._delay == backoff_delay - def test_rampup_step_held_during_backoff(self): - scope = _scope_manager(config={"id": "x", "delay": 4.0, "rampup": True}) - scope._backoff_level = 1 - before = scope._delay - scope._rampup_step() - # A rampup probe is skipped while a backoff is in progress. - assert scope._delay == before - def test_record_sent_clears_expired_backoff(self): scope = _scope_manager(config={"id": "x"}) scope.record_backoff(delay=5.0, now=0.0) From bce275c1dcba17f0a3facd7e82b4896e8c8f9aeb Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 12:31:54 +0200 Subject: [PATCH 52/55] Human review --- scrapy/core/engine.py | 2 +- scrapy/core/scheduler.py | 155 +++++++++++++++------------------------ tests/test_scheduler.py | 23 ------ 3 files changed, 62 insertions(+), 118 deletions(-) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 217d52826..2dc64c5c3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -539,7 +539,7 @@ class ExecutionEngine: self._slot.nextcall.schedule() # type: ignore[union-attr] def _schedule_request(self, request: Request) -> None: - assert self._slot is not None # typing + assert self._slot is not None request_scheduled_result = self.signals.send_catch_log( signals.request_scheduled, request=request, diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 78d13bcb4..fcdb67cab 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -65,6 +65,33 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): plays a great part in determining the order in which those requests are downloaded. See :ref:`request-order`. The methods defined in this class constitute the minimal interface that the Scrapy engine will interact with. + + Asynchronous API + ================ + + Beyond the minimal interface above, the engine also uses the following + optional members when a scheduler defines them, to support asynchronous + scheduling. :meth:`next_request` itself stays synchronous. + + .. method:: enqueue_request_async(request) + :async: + + Asynchronous counterpart of :meth:`enqueue_request`, following the same + return contract. When a scheduler defines it, the engine awaits it + *instead of* calling :meth:`enqueue_request`. Define it when enqueuing a + request needs to ``await`` (e.g. to resolve throttling scopes + asynchronously). + + .. method:: get_next_request_delay() + + Return the number of seconds until some pending request that + :meth:`next_request` is currently withholding for a time-based reason + becomes available, or ``None`` if no pending request is time-blocked. + When a scheduler defines it, the engine uses it to schedule a single + wakeup after :meth:`next_request` returns ``None`` while requests remain + pending, so a crawl held back only by time does not stall. + + .. versionadded:: VERSION """ @classmethod @@ -375,12 +402,21 @@ class Scheduler(BaseScheduler): if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False - dqok = self._dqpush(request) + return self._store(request) + + def _store(self, request: Request, *push_args: Any) -> bool: + """Push *request* into the disk queue, falling back to the memory queue, + and increment the relevant ``scheduler/enqueued*`` stats. + + Extra positional arguments are forwarded to the underlying queue + ``push`` calls; subclasses that store requests under additional keys + (e.g. a throttling scope set) pass them through here. + """ assert self.stats is not None - if dqok: + if self._dqpush(request, *push_args): self.stats.inc_value("scheduler/enqueued/disk") else: - self._mqpush(request) + self._mqpush(request, *push_args) self.stats.inc_value("scheduler/enqueued/memory") self.stats.inc_value("scheduler/enqueued") return True @@ -412,11 +448,11 @@ class Scheduler(BaseScheduler): """ return len(self.dqs) + len(self.mqs) if self.dqs is not None else len(self.mqs) - def _dqpush(self, request: Request) -> bool: + def _dqpush(self, request: Request, *push_args: Any) -> bool: if self.dqs is None: return False try: - self.dqs.push(request) + self.dqs.push(request, *push_args) except ValueError as e: # non serializable request if self.logunser: msg = ( @@ -436,8 +472,8 @@ class Scheduler(BaseScheduler): return False return True - def _mqpush(self, request: Request) -> None: - self.mqs.push(request) + def _mqpush(self, request: Request, *push_args: Any) -> None: + self.mqs.push(request, *push_args) def _dqpop(self) -> Request | None: if self.dqs is not None: @@ -499,37 +535,25 @@ class Scheduler(BaseScheduler): json.dump(state, f) -def _queue_supports_peek(queue_cls: type) -> bool: - """Return whether *queue_cls* (a Scrapy queue class) is backed by an - underlying queue that really implements ``peek``. - - Scrapy's queue wrappers in :mod:`scrapy.squeues` always define a ``peek`` - method, but it merely delegates to the underlying queue class (e.g. a - queuelib one, which only gained ``peek`` in queuelib 1.6.1) and raises - :exc:`NotImplementedError` if that one lacks it. This checks the real - implementation by ignoring the delegating wrappers. - """ - return any( - "peek" in base.__dict__ and base.__module__ != "scrapy.squeues" - for base in queue_cls.__mro__ - ) - - class ThrottlingAwareScheduler(Scheduler): - """A :setting:`SCHEDULER` that only ever hands the engine requests whose - :ref:`throttling scopes ` allow them to be sent **right + """A :setting:`SCHEDULER` that only ever hands the engine requests that + their :ref:`throttling scopes ` allow to be sent **right now**. - This avoids the head-of-line blocking that the default scheduler can suffer - when a crawl mixes heavily-throttled scopes with unthrottled ones: with the - default scheduler, requests taken from the scheduler in order wait at the - throttling gate - (:meth:`~scrapy.throttling.ThrottlingManagerProtocol.acquire`) while - holding a concurrency slot, so enough throttled requests waiting on a clock - can fill :setting:`CONCURRENT_REQUESTS` and starve unthrottled requests - that could be sent right away. Requests held back by this scheduler instead - occupy neither concurrency slots nor memory (beyond what is needed to track - each distinct scope set), so they cannot starve other scopes. + The default scheduler hands requests to the engine as concurrency allows + and lets them wait at the throttling gate + (:meth:`~scrapy.throttling.ThrottlingManagerProtocol.acquire`) while holding + a concurrency slot. When a crawl mixes heavily-throttled scopes with + unthrottled ones, enough throttled requests waiting on a clock can fill + :setting:`CONCURRENT_REQUESTS` and starve unthrottled requests that could be + sent right away. This scheduler instead withholds a request until it is + sendable, so held-back requests occupy neither concurrency slots nor memory + (beyond what is needed to track each distinct scope set) and cannot starve + other scopes. + + It also honors the per-request :reqmeta:`throttling_delay`, holding an + individual request back without blocking others that share its scopes, which + the default scheduler cannot do. When several requests could be sent at the same time, the one with the highest request :attr:`~scrapy.Request.priority` is sent first; ties are @@ -537,8 +561,7 @@ class ThrottlingAwareScheduler(Scheduler): It requires :setting:`SCHEDULER_PRIORITY_QUEUE` to be set to :class:`~scrapy.pqueues.ThrottlingAwarePriorityQueue` (or a compatible - subclass), and the configured :setting:`SCHEDULER_DISK_QUEUE` / - :setting:`SCHEDULER_MEMORY_QUEUE` to support ``peek``. + subclass). """ def open(self, spider: Spider) -> Deferred[None] | None: @@ -550,19 +573,6 @@ class ThrottlingAwareScheduler(Scheduler): f"scrapy.pqueues.ThrottlingAwarePriorityQueue, but the " f"configured one ({type(self.mqs).__name__}) is not." ) - for setting, pq in ( - ("SCHEDULER_MEMORY_QUEUE", self.mqs), - ("SCHEDULER_DISK_QUEUE", self.dqs), - ): - if pq is None: - continue - queue_cls = getattr(pq, "downstream_queue_cls", None) - if queue_cls is not None and not _queue_supports_peek(queue_cls): - raise ValueError( - f"{type(self).__name__} requires {setting} to be set to a " - f"queue class that supports peek(), but the configured one " - f"({queue_cls.__name__}) does not." - ) assert self.crawler is not None assert self.crawler.throttler is not None self._throttler: ThrottlingManagerProtocol = self.crawler.throttler @@ -576,56 +586,13 @@ class ThrottlingAwareScheduler(Scheduler): ) async def enqueue_request_async(self, request: Request) -> bool: - """Resolve the :ref:`throttling scope set ` of - *request* and store it under the queue for that scope set. - - This is the asynchronous counterpart of - :meth:`Scheduler.enqueue_request`; scope resolution - (:meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scopes`) is - asynchronous, which is why enqueuing has to be too. - """ if not request.dont_filter and self.df.request_seen(request): self.df.log(request, self.spider) return False scope_set = frozenset(iter_scopes(await self._throttler.get_scopes(request))) - assert self.stats is not None - if self._dqpush_throttling(request, scope_set): - self.stats.inc_value("scheduler/enqueued/disk") - else: - self.mqs.push(request, scope_set) # type: ignore[call-arg] - self.stats.inc_value("scheduler/enqueued/memory") - self.stats.inc_value("scheduler/enqueued") - return True - - def _dqpush_throttling(self, request: Request, scope_set: frozenset[str]) -> bool: - if self.dqs is None: - return False - try: - self.dqs.push(request, scope_set) # type: ignore[call-arg] - except ValueError as e: # non serializable request - if self.logunser: - logger.warning( - "Unable to serialize request: %(request)s - reason:" - " %(reason)s - no more unserializable requests will be" - " logged (stats being collected)", - {"request": request, "reason": e}, - exc_info=True, - extra={"spider": self.spider}, - ) - self.logunser = False - assert self.stats is not None - self.stats.inc_value("scheduler/unserializable") - return False - return True + return self._store(request, scope_set) def get_next_request_delay(self) -> float | None: - """Return the minimum number of seconds until some pending request - becomes sendable because a time-based throttling gate opens, or - ``None`` if no pending request is time-blocked. - - The engine uses this to arm a single wakeup timer when - :meth:`next_request` returns ``None`` while requests are still - pending.""" delays = [ delay for pq in (self.mqs, self.dqs) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index d1d56bbcb..b8b1f8c7d 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -27,9 +27,6 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path - # typing.Self requires Python 3.11 - from typing_extensions import Self - from scrapy.http.request import CallbackT @@ -417,16 +414,6 @@ class TestIncompatibility: _THROTTLING_AWARE_PQ = "scrapy.pqueues.ThrottlingAwarePriorityQueue" -class _NoPeekMemoryQueue: - """A memory queue class that does not implement ``peek``, used to check - that ThrottlingAwareScheduler rejects queues lacking peek support (e.g. when - queuelib is older than 1.6.1).""" - - @classmethod - def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: - return cls() - - class TestThrottlingAwareScheduler: def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler: settings = { @@ -472,16 +459,6 @@ class TestThrottlingAwareScheduler: with pytest.raises(ValueError, match="throttling-aware priority queue"): scheduler.open(spider) - def test_requires_peek_supporting_queue(self) -> None: - crawler = self._crawler( - {"SCHEDULER_MEMORY_QUEUE": "tests.test_scheduler._NoPeekMemoryQueue"} - ) - spider = Spider(name="spider") - crawler.spider = spider - scheduler = ThrottlingAwareScheduler.from_crawler(crawler) - with pytest.raises(ValueError, match="supports peek"): - scheduler.open(spider) - @coroutine_test async def test_delay_blocks_and_reports_delay(self) -> None: crawler = self._crawler( From ca8cc4630ace0895761a942986824b57acc9b56f Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 12:50:19 +0200 Subject: [PATCH 53/55] Human review --- docs/topics/throttling.rst | 24 ++++--- scrapy/downloadermiddlewares/backoff.py | 29 +++++--- scrapy/extensions/throttle.py | 4 +- .../templates/project/module/settings.py.tmpl | 68 +++++++++++++++++-- scrapy/throttling.py | 31 ++------- scrapy/utils/test.py | 12 ++-- 6 files changed, 107 insertions(+), 61 deletions(-) diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index d087ab302..06799a7f4 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -752,8 +752,7 @@ to: the API itself. The API returns ``200`` even when the target website rate-limits it, reporting the upstream status in a header; the middleware backs off the **target-website** scope (not the API scope) in that case, - reusing the scope's own :meth:`~scrapy.throttling.ThrottlingScopeManagerProtocol.triggers_backoff_for_status` - check: + checking the upstream status against :setting:`BACKOFF_HTTP_CODES`: .. code-block:: python @@ -764,6 +763,9 @@ to: class UpstreamBackoffMiddleware: def __init__(self, crawler): self.throttler = crawler.throttler + self.backoff_codes = { + int(code) for code in crawler.settings.getlist("BACKOFF_HTTP_CODES") + } @classmethod def from_crawler(cls, crawler): @@ -774,15 +776,15 @@ to: upstream_status = int( response.headers.get("X-Upstream-Status-Code", b"200") ) - scopes = [ - scope - for scope in iter_scopes(self.throttler.get_resolved_scopes(request)) - if scope != "api.toscrape.com" - and self.throttler.get_scope_manager(scope).triggers_backoff_for_status( - upstream_status - ) - ] - self.throttler.back_off(scopes) + if upstream_status in self.backoff_codes: + scopes = [ + scope + for scope in iter_scopes( + self.throttler.get_resolved_scopes(request) + ) + if scope != "api.toscrape.com" + ] + self.throttler.back_off(scopes) return response diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 4925973ed..80c9eb140 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -54,12 +54,23 @@ class BackoffMiddleware: self._exceptions: tuple[type[BaseException], ...] = _load_objects( settings.getlist("BACKOFF_EXCEPTIONS") ) - for scope_config in settings.getdict("THROTTLING_SCOPES").values(): + # Per-scope overrides; a scope absent from these uses the globals above. + self._scope_http_codes: dict[str, set[int]] = {} + self._scope_exceptions: dict[str, tuple[type[BaseException], ...]] = {} + # Unions of the globals and every per-scope override, used as a cheap + # pre-filter to skip outcomes that no scope backs off on. + self._any_http_codes: set[int] = set(self._http_codes) + self._any_exceptions: tuple[type[BaseException], ...] = self._exceptions + for scope_id, scope_config in settings.getdict("THROTTLING_SCOPES").items(): backoff = scope_config.get("backoff") or {} if "http_codes" in backoff: - self._http_codes.update(int(code) for code in backoff["http_codes"]) + codes = {int(code) for code in backoff["http_codes"]} + self._scope_http_codes[scope_id] = codes + self._any_http_codes |= codes if "exceptions" in backoff: - self._exceptions += _load_objects(backoff["exceptions"]) + exceptions = _load_objects(backoff["exceptions"]) + self._scope_exceptions[scope_id] = exceptions + self._any_exceptions += exceptions @_warn_spider_arg def process_response( @@ -69,7 +80,7 @@ class BackoffMiddleware: spider: scrapy.Spider | None = None, ) -> Response: if ( - response.status not in self._http_codes + response.status not in self._any_http_codes or "cached" in response.flags or request.meta.get("dont_throttle") ): @@ -77,9 +88,7 @@ class BackoffMiddleware: matched = [ scope for scope in iter_scopes(self._throttler.get_resolved_scopes(request)) - if self._throttler.get_scope_manager(scope).triggers_backoff_for_status( - response.status - ) + if response.status in self._scope_http_codes.get(scope, self._http_codes) ] if matched: self._throttler.back_off(matched, delay=self._response_delay(response)) @@ -93,14 +102,14 @@ class BackoffMiddleware: spider: scrapy.Spider | None = None, ) -> None: if request.meta.get("dont_throttle") or not isinstance( - exception, self._exceptions + exception, self._any_exceptions ): return matched = [ scope for scope in iter_scopes(self._throttler.get_resolved_scopes(request)) - if self._throttler.get_scope_manager(scope).triggers_backoff_for_exception( - exception + if isinstance( + exception, self._scope_exceptions.get(scope, self._exceptions) ) ] if matched: diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 93f82268f..9f3736c5b 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -57,7 +57,7 @@ class AutoThrottle: def _spider_opened(self, spider: Spider) -> None: self.mindelay = self._min_delay(spider) self.maxdelay = self._max_delay(spider) - self.startdelay = max( + self._startdelay = max( self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY") ) @@ -107,7 +107,7 @@ class AutoThrottle: delay = throttler.get_scope_delay(scope_id) if scope_id not in self._started_scopes: self._started_scopes.add(scope_id) - delay = max(delay, self.startdelay) + delay = max(delay, self._startdelay) return delay def _adjust_delay( diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index b9faac9a5..6a120aa47 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -1,18 +1,28 @@ -# https://docs.scrapy.org/en/latest/topics/settings.html +# Scrapy settings for $project_name project +# +# For simplicity, this file contains only settings considered important or +# commonly used. You can find more settings consulting the documentation: +# +# https://docs.scrapy.org/en/latest/topics/settings.html +# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +# https://docs.scrapy.org/en/latest/topics/spider-middleware.html BOT_NAME = "$project_name" SPIDER_MODULES = ["$project_name.spiders"] NEWSPIDER_MODULE = "$project_name.spiders" -# Crawl responsibly by identifying yourself (and your website) through the -# User-Agent header: -#USER_AGENT = "$project_name (+https://your-domain.example)" +ADDONS = {} -# Obey robots.txt rules: + +# Crawl responsibly by identifying yourself (and your website) on the user-agent +#USER_AGENT = "$project_name (+http://www.yourdomain.com)" + +# Obey robots.txt rules ROBOTSTXT_OBEY = True -# Throttle crawls to be polite to websites: +# Concurrency and throttling settings +#CONCURRENT_REQUESTS = 16 THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 @@ -22,5 +32,49 @@ DOWNLOAD_DELAY = 1 # "example.com": {"concurrency": 16, "delay": 0.1}, #} -# Set settings whose default value is deprecated to a future-proof value: +# Disable cookies (enabled by default) +#COOKIES_ENABLED = False + +# Disable Telnet Console (enabled by default) +#TELNETCONSOLE_ENABLED = False + +# Override the default request headers: +#DEFAULT_REQUEST_HEADERS = { +# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", +# "Accept-Language": "en", +#} + +# Enable or disable spider middlewares +# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html +#SPIDER_MIDDLEWARES = { +# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543, +#} + +# Enable or disable downloader middlewares +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html +#DOWNLOADER_MIDDLEWARES = { +# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543, +#} + +# Enable or disable extensions +# See https://docs.scrapy.org/en/latest/topics/extensions.html +#EXTENSIONS = { +# "scrapy.extensions.telnet.TelnetConsole": None, +#} + +# Configure item pipelines +# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html +#ITEM_PIPELINES = { +# "$project_name.pipelines.${ProjectName}Pipeline": 300, +#} + +# Enable and configure HTTP caching (disabled by default) +# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings +#HTTPCACHE_ENABLED = True +#HTTPCACHE_EXPIRATION_SECS = 0 +#HTTPCACHE_DIR = "httpcache" +#HTTPCACHE_IGNORE_HTTP_CODES = [] +#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" + +# Set settings whose default value is deprecated to a future-proof value FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/throttling.py b/scrapy/throttling.py index e460a5163..6785d607e 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -19,7 +19,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import SETTINGS_PRIORITIES from scrapy.utils.asyncio import sleep, wait_for_first from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import _load_objects, build_from_crawler, load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: from scrapy.crawler import Crawler @@ -1031,14 +1031,6 @@ class ThrottlingScopeManagerProtocol(Protocol): reported for a request, correcting the estimate used by :meth:`record_sent`.""" - def triggers_backoff_for_status(self, status: int) -> bool: - """Return whether a response with the given *status* triggers backoff - for this scope (defaults to :setting:`BACKOFF_HTTP_CODES`).""" - - def triggers_backoff_for_exception(self, exception: BaseException) -> bool: - """Return whether *exception* triggers backoff for this scope (defaults - to :setting:`BACKOFF_EXCEPTIONS`).""" - def get_base_delay(self) -> float: """Return the base (non-backoff) delay of this scope, in seconds.""" @@ -1167,18 +1159,9 @@ class ThrottlingScopeManager: self._backoff_jitter: tuple[float, float] | None = self._normalize_jitter( backoff.get("jitter", settings.getfloat("BACKOFF_JITTER")) ) - # Which responses/exceptions trigger backoff for this scope. Each - # defaults to the matching global BACKOFF_* setting (see - # triggers_backoff_for_status / triggers_backoff_for_exception). - self._backoff_http_codes: set[int] = { - int(code) - for code in backoff.get( - "http_codes", settings.getlist("BACKOFF_HTTP_CODES") - ) - } - self._backoff_exceptions: tuple[type[BaseException], ...] = _load_objects( - backoff.get("exceptions", settings.getlist("BACKOFF_EXCEPTIONS")) - ) + # Which responses/exceptions trigger backoff is decided by the backoff + # middleware (see BackoffMiddleware), which reads the same per-scope + # "http_codes"/"exceptions" config and the global BACKOFF_* settings. self._window: float = settings.getfloat("BACKOFF_WINDOW") # Concurrency. ``None`` means no scope-level limit (the downloader slots @@ -1399,12 +1382,6 @@ class ThrottlingScopeManager: elif consumed is not None: self._consumed = max(0.0, self._consumed + float(consumed)) - def triggers_backoff_for_status(self, status: int) -> bool: - return status in self._backoff_http_codes - - def triggers_backoff_for_exception(self, exception: BaseException) -> bool: - return isinstance(exception, self._backoff_exceptions) - def get_base_delay(self) -> float: return self._base_delay diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index ee289b4e9..4b69733dc 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -15,6 +15,7 @@ from twisted.web.client import Agent from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.settings import default_settings from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed from scrapy.utils.spider import DefaultSpider @@ -72,10 +73,13 @@ def get_crawler( **(settings_dict or {}), } if prevent_warnings: - # Pin the pre-deprecation default per-scope concurrency so the suite - # keeps its historical behavior and does not emit the warn-then-flip - # deprecation warning (see throttling._warn_on_deprecated_concurrency). - settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8) + # Pin the per-scope concurrency to CONCURRENT_REQUESTS_PER_DOMAIN's + # default: keeps the suite's historical behavior and silences the + # warn-then-flip warning (see throttling._warn_on_deprecated_concurrency). + settings.setdefault( + "THROTTLING_SCOPE_CONCURRENCY", + default_settings.CONCURRENT_REQUESTS_PER_DOMAIN, + ) runner: CrawlerRunnerBase if is_reactor_installed(): runner = CrawlerRunner(settings) From 8fcb5447659b9c34722ab6adc287f561b79f5bb5 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 13:17:36 +0200 Subject: [PATCH 54/55] Remove THROTTLING_SCOPES from the settings.py template --- scrapy/templates/project/module/settings.py.tmpl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 6a120aa47..b010a7860 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -26,12 +26,6 @@ ROBOTSTXT_OBEY = True THROTTLING_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 -# Override the polite defaults above for specific domains, e.g. to crawl a site -# you own (or one meant for scraping) faster: -#THROTTLING_SCOPES = { -# "example.com": {"concurrency": 16, "delay": 0.1}, -#} - # Disable cookies (enabled by default) #COOKIES_ENABLED = False From f0cc534737a2a460fff2e3c946cb20fbac90207b Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 3 Jul 2026 16:18:36 +0200 Subject: [PATCH 55/55] =?UTF-8?q?throttling=20=E2=86=92=20throttler?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/intro/tutorial.rst | 4 +- docs/topics/broad-crawls.rst | 2 +- docs/topics/components.rst | 4 +- docs/topics/request-response.rst | 4 +- docs/topics/throttling.rst | 319 +++++++++--------- scrapy/core/downloader/__init__.py | 19 +- scrapy/core/downloader/handlers/_base_http.py | 6 +- scrapy/core/engine.py | 42 +-- scrapy/core/scheduler.py | 28 +- scrapy/crawler.py | 16 +- scrapy/downloadermiddlewares/backoff.py | 12 +- scrapy/downloadermiddlewares/redirect.py | 8 +- scrapy/extensions/throttle.py | 8 +- scrapy/pqueues.py | 68 ++-- scrapy/settings/default_settings.py | 40 +-- .../templates/project/module/settings.py.tmpl | 3 +- scrapy/{throttling.py => throttler.py} | 212 ++++++------ scrapy/utils/test.py | 4 +- tests/test_crawler.py | 14 +- tests/test_engine.py | 26 +- tests/test_pqueues.py | 103 +++--- tests/test_scheduler.py | 44 +-- .../{test_throttling.py => test_throttler.py} | 156 +++++---- 23 files changed, 564 insertions(+), 578 deletions(-) rename scrapy/{throttling.py => throttler.py} (88%) rename tests/{test_throttling.py => test_throttler.py} (90%) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 94e27386c..cb9118304 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -141,12 +141,12 @@ parallel. ``quotes.toscrape.com`` is a sandbox meant for scraping practice, so we can safely crawl it faster. Open ``tutorial/settings.py`` and add a -:setting:`THROTTLING_SCOPES` entry that raises the concurrency and lowers the +:setting:`THROTTLER_SCOPES` entry that raises the concurrency and lowers the delay for that domain only: .. code-block:: python - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "quotes.toscrape.com": {"concurrency": 16, "delay": 0.1}, } diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 21096bdfb..ffcff60ae 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -48,7 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional per-scope -(per-domain by default) limit (:setting:`THROTTLING_SCOPE_CONCURRENCY`). +(per-domain by default) limit (:setting:`THROTTLER_SCOPE_CONCURRENCY`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d7aa5642a..8f9b3865a 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -24,8 +24,8 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_START_DISK_QUEUE` - :setting:`SCHEDULER_START_MEMORY_QUEUE` - :setting:`SPIDER_MIDDLEWARES` -- :setting:`THROTTLING_MANAGER` -- :setting:`THROTTLING_SCOPE_MANAGER` +- :setting:`THROTTLER` +- :setting:`THROTTLER_SCOPE_MANAGER` - :setting:`TWISTED_DNS_RESOLVER` Third-party Scrapy components may also let you define additional Scrapy diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f4d2d0eaa..a9fd6fd87 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -733,8 +733,8 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` -* :reqmeta:`throttling_delay` -* :reqmeta:`throttling_scopes` +* :reqmeta:`delay` +* :reqmeta:`throttler_scopes` * :reqmeta:`verbatim_url` .. reqmeta:: bindaddress diff --git a/docs/topics/throttling.rst b/docs/topics/throttling.rst index 06799a7f4..34b4f520b 100644 --- a/docs/topics/throttling.rst +++ b/docs/topics/throttling.rst @@ -26,12 +26,12 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``. The main throttling :ref:`settings ` are: -- .. setting:: THROTTLING_SCOPE_CONCURRENCY +- .. setting:: THROTTLER_SCOPE_CONCURRENCY - :setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``) + :setting:`THROTTLER_SCOPE_CONCURRENCY` (default: ``1``) - Default maximum number of simultaneous requests per :ref:`throttling scope - `. Requests are grouped by domain by default, so this is + Default maximum number of simultaneous requests per :ref:`throttler scope + `. Requests are grouped by domain by default, so this is the maximum number of simultaneous requests per domain. It defines a number of “slots” per scope. Each slot can send 1 request at a @@ -51,29 +51,29 @@ The main throttling :ref:`settings ` are: When configuring these settings, note that: -- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`. +- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLER_SCOPE_CONCURRENCY`. - If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``, because the next request to the domain is not sent until the delay elapses, by which time the previous response has already arrived. -.. [1] You can :ref:`customize ` how requests are grouped +.. [1] You can :ref:`customize ` how requests are grouped for throttling, but domain-based throttling works well in most cases. For more complex domain grouping strategies, see :ref:`alternative-domain-throttling`. -.. setting:: THROTTLING_SCOPES +.. setting:: THROTTLER_SCOPES .. _per-domain-throttling: Per-domain throttling ===================== -The :setting:`THROTTLING_SCOPES` setting allows you to customize throttling +The :setting:`THROTTLER_SCOPES` setting allows you to customize throttling behavior for specific domains [1]_. It is a dict that maps scope names to -:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts. It is empty by default. +:class:`~scrapy.throttler.ThrottlerScopeConfig` dicts. It is empty by default. :command:`startproject` scaffolds a commented-out example entry, so that you can uncomment and edit it to crawl domains you own (or that are meant for scraping) @@ -82,7 +82,7 @@ other domains: .. code-block:: python - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "example.com": {"concurrency": 16, "delay": 0.1}, } @@ -133,7 +133,7 @@ See :ref:`throttling-settings` for :setting:`BACKOFF_EXCEPTIONS`, How backoff works ----------------- -Every :ref:`throttling scope ` keeps a current delay that +Every :ref:`throttler scope ` keeps a current delay that starts at its configured ``"delay"`` (:setting:`DOWNLOAD_DELAY` by default). A **backoff trigger** is a response whose status code is in @@ -181,7 +181,7 @@ Backoff triggers are detected by the Any component can also trigger backoff programmatically for arbitrary scopes — e.g. based on the response body of a specific site — through :meth:`crawler.throttler.back_off() -`. +`. .. _per-scope-backoff: @@ -189,13 +189,13 @@ Per-scope backoff configuration ------------------------------- The global ``BACKOFF_*`` settings can be overridden per scope with the -``"backoff"`` key of a :setting:`THROTTLING_SCOPES` entry, an instance of -:class:`~scrapy.throttling.BackoffConfig`: +``"backoff"`` key of a :setting:`THROTTLER_SCOPES` entry, an instance of +:class:`~scrapy.throttler.BackoffConfig`: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "example.com": { "backoff": { "http_codes": [429, 503], @@ -242,17 +242,17 @@ robots.txt is a non-standard ``robots.txt`` directive that indicates a number of seconds to wait between requests. -.. setting:: THROTTLING_ROBOTSTXT_OBEY -.. setting:: THROTTLING_ROBOTSTXT_MAX_DELAY +.. setting:: THROTTLER_ROBOTSTXT_OBEY +.. setting:: THROTTLER_ROBOTSTXT_MAX_DELAY -If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are +If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLER_ROBOTSTXT_OBEY` are ``True`` (default), valid ``Crawl-Delay`` directives override -:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. +:setting:`THROTTLER_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency is set to ``1`` and the delay is raised to at least the ``Crawl-Delay`` value (a larger configured delay is kept), capped at -:setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). +:setting:`THROTTLER_ROBOTSTXT_MAX_DELAY` (default: ``60.0``). -If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it +If :setting:`THROTTLER_SCOPES` defines a different concurrency or delay, it will be respected, but a warning will be logged about the discrepancy with ``Crawl-Delay``. Set ``ignore_robots_txt`` to ``True`` to silence this warning. @@ -261,9 +261,9 @@ will be respected, but a warning will be logged about the discrepancy with Delaying a scope programmatically ================================= -You can delay a :ref:`throttling scope ` on demand through +You can delay a :ref:`throttler scope ` on demand through :meth:`crawler.throttler.delay_scope() -`: +`: .. skip: next @@ -315,9 +315,9 @@ For example, you might want to throttle API endpoints differently than web pages on the same domain, group requests by content type (images vs HTML), or apply different throttling based on request priority. -.. reqmeta:: throttling_scopes +.. reqmeta:: throttler_scopes -Use the ``throttling_scopes`` request metadata to assign requests to custom +Use the ``throttler_scopes`` request metadata to assign requests to custom throttling groups: .. invisible-code-block: python @@ -326,51 +326,51 @@ throttling groups: .. code-block:: python - Request("https://api.example/", meta={"throttling_scopes": "api"}) + Request("https://api.example/", meta={"throttler_scopes": "api"}) You can also assign multiple throttling groups to a single request: .. code-block:: python - Request("https://api.example/users", meta={"throttling_scopes": {"api", "users"}}) + Request("https://api.example/users", meta={"throttler_scopes": {"api", "users"}}) -You can then use the :setting:`THROTTLING_SCOPES` setting to customize +You can then use the :setting:`THROTTLER_SCOPES` setting to customize throttling for such requests: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "api": {"concurrency": 2}, "users": {"delay": 5.0}, } .. note:: These custom throttling groups persist through redirects. For - redirect-aware throttling assignment, see :ref:`custom-throttling-scopes`. + redirect-aware throttling assignment, see :ref:`custom-throttler-scopes`. -.. reqmeta:: throttling_delay +.. reqmeta:: delay Delaying a single request ------------------------- To hold a single request for a fixed number of seconds before it is sent, -regardless of its scopes, set the ``throttling_delay`` request metadata key: +regardless of its scopes, set the ``delay`` request metadata key: .. code-block:: python - Request("https://example.com/slow", meta={"throttling_delay": 5.0}) + Request("https://example.com/slow", meta={"delay": 5.0}) The delay is applied once, the first time the request reaches the throttling gate. -``throttling_delay`` defines only the *earliest* time the request may be sent, +``delay`` defines only the *earliest* time the request may be sent, not the exact time: once the delay elapses, the request still competes with every other pending request for its scopes. If you want it sent **as soon as** its delay elapses, give it a higher :attr:`~scrapy.Request.priority` too: .. code-block:: python - Request("https://example.com/slow", meta={"throttling_delay": 5.0}, priority=1) + Request("https://example.com/slow", meta={"delay": 5.0}, priority=1) Without a higher priority, a backlog of requests ahead of it in a FIFO queue could keep it waiting well past the configured delay; a higher priority puts it @@ -391,33 +391,33 @@ request normally without letting its outcome trigger :ref:`backoff `: Request("https://example.com/login", meta={"dont_throttle": True}) -.. _throttling-scopes: +.. _throttler-scopes: -Throttling scopes -================= +Throttler scopes +================ -Throttling scopes represent aspects of requests that can be throttled +Throttler scopes represent aspects of requests that can be throttled independently. .. - For future reference, the “throttling scope” name was taken from + For future reference, the “throttler scope” name was taken from https://www.ietf.org/archive/id/draft-polli-ratelimit-headers-02.html#section-1.4-4.4 -.. _custom-throttling-scopes: +.. _custom-throttler-scopes: -Customizing throttling scopes +Customizing throttler scopes ------------------------------ -There are 2 ways to customize throttling scopes. +There are 2 ways to customize throttler scopes. -To **configure existing scopes**, use the :setting:`THROTTLING_SCOPES` setting. +To **configure existing scopes**, use the :setting:`THROTTLER_SCOPES` setting. Its keys are scope names and its values are -:class:`~scrapy.throttling.ThrottlingScopeConfig` dicts, which accept the +:class:`~scrapy.throttler.ThrottlerScopeConfig` dicts, which accept the following keys: ``concurrency`` (:class:`int`) Maximum number of concurrent requests for the scope. Defaults to - :setting:`THROTTLING_SCOPE_CONCURRENCY`. + :setting:`THROTTLER_SCOPE_CONCURRENCY`. ``delay`` (:class:`float`) Minimum seconds between requests for the scope. Defaults to @@ -430,152 +430,152 @@ following keys: a ``[low, high]`` pair multiplies the delay by ``1 + uniform(low, high)``. ``quota`` (:class:`float`) - Maximum :ref:`quota ` consumed per ``window``. + Maximum :ref:`quota ` consumed per ``window``. ``window`` (:class:`float`) - Quota window in seconds. Defaults to :setting:`THROTTLING_WINDOW`. + Quota window in seconds. Defaults to :setting:`THROTTLER_WINDOW`. -``backoff`` (:class:`~scrapy.throttling.BackoffConfig`) +``backoff`` (:class:`~scrapy.throttler.BackoffConfig`) Per-scope :ref:`backoff overrides `. ``manager`` (:class:`str` or :class:`type`) Import path or class of a :ref:`custom scope manager - ` for the scope. + ` for the scope. ``ignore_robots_txt`` (:class:`bool`) Silences the warning logged when this configuration is more aggressive than a :ref:`robots.txt Crawl-delay `. -.. setting:: THROTTLING_MANAGER +.. setting:: THROTTLER To **change how scopes are assigned** (or anything beyond per-scope settings), -set :setting:`THROTTLING_MANAGER` (default: -:class:`~scrapy.throttling.ThrottlingManager`) to a :ref:`component +set :setting:`THROTTLER` (default: +:class:`~scrapy.throttler.Throttler`) to a :ref:`component ` that implements the -:class:`~scrapy.throttling.ThrottlingManagerProtocol` protocol (or its import +:class:`~scrapy.throttler.ThrottlerProtocol` protocol (or its import path as a string): .. code-block:: python :caption: ``settings.py`` - THROTTLING_MANAGER = "myproject.throttling.MyThrottlingManager" + THROTTLER = "myproject.throttling.MyThrottler" -.. _multiple-throttling-scopes: +.. _multiple-throttler-scopes: -Handling of multiple throttling scopes +Handling of multiple throttler scopes -------------------------------------- -When a request has multiple throttling scopes, it is not sent until all of its -throttling scopes allow it. +When a request has multiple throttler scopes, it is not sent until all of its +throttler scopes allow it. -.. _throttling-quotas: +.. _throttler-quotas: -Throttling quotas ------------------ +Throttler quotas +---------------- -When different requests can consume different amounts of a throttling scope, -you can express this using **throttling quotas**. +When different requests can consume different amounts of a throttler scope, +you can express this using **throttler quotas**. -.. setting:: THROTTLING_WINDOW +.. setting:: THROTTLER_WINDOW -Use the :setting:`THROTTLING_WINDOW` setting (default: ``60.0``) or the ``"window"`` -key in the :setting:`THROTTLING_SCOPES` setting to define the time window after -which throttling quotas are reset. +Use the :setting:`THROTTLER_WINDOW` setting (default: ``60.0``) or the ``"window"`` +key in the :setting:`THROTTLER_SCOPES` setting to define the time window after +which throttler quotas are reset. -Then use the :setting:`THROTTLING_SCOPES` setting to define the throttling -quotas for each throttling scope: +Then use the :setting:`THROTTLER_SCOPES` setting to define the throttling +quotas for each throttler scope: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "api.toscrape.com": { "quota": 500.0, }, } -Then, in the :reqmeta:`throttling_scopes` request metadata key or in the return -value of the :meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scopes` -method, define a :class:`dict` where keys are throttling scopes and values are +Then, in the :reqmeta:`throttler_scopes` request metadata key or in the return +value of the :meth:`~scrapy.throttler.ThrottlerProtocol.get_scopes` +method, define a :class:`dict` where keys are throttler scopes and values are :class:`float` values that indicate the expected quota consumption (it does not need to be exact). -.. _custom-throttling-scope-managers: +.. _custom-throttler-scope-managers: -Customizing throttling scope managers +Customizing throttler scope managers ------------------------------------- -.. setting:: THROTTLING_SCOPE_MANAGER +.. setting:: THROTTLER_SCOPE_MANAGER -The :setting:`THROTTLING_SCOPE_MANAGER` setting (default: -:class:`~scrapy.throttling.ThrottlingScopeManager`) is a :ref:`component +The :setting:`THROTTLER_SCOPE_MANAGER` setting (default: +:class:`~scrapy.throttler.ThrottlerScopeManager`) is a :ref:`component ` that implements the -:class:`~scrapy.throttling.ThrottlingScopeManagerProtocol` (or its import path +:class:`~scrapy.throttler.ThrottlerScopeManagerProtocol` (or its import path as a string): .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPE_MANAGER = "myproject.throttling.MyThrottlingScopeManager" + THROTTLER_SCOPE_MANAGER = "myproject.throttling.MyThrottlerScopeManager" -For each throttling scope, an instance of this class is created to manage any +For each throttler scope, an instance of this class is created to manage any gradual :ref:`backoff ` required at run time. -You can implement your own throttling scope manager if you wish to change the +You can implement your own throttler scope manager if you wish to change the backoff behavior beyond what settings allow. -You can also define a custom throttling scope manager for a specific throttling -scope by setting the ``"manager"`` key in the :setting:`THROTTLING_SCOPES` +You can also define a custom throttler scope manager for a specific throttling +scope by setting the ``"manager"`` key in the :setting:`THROTTLER_SCOPES` setting: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "api.toscrape.com": { - "manager": "myproject.throttling.MyThrottlingScopeManager", + "manager": "myproject.throttling.MyThrottlerScopeManager", }, } Most custom scope managers subclass the default -:class:`~scrapy.throttling.ThrottlingScopeManager` and override only the methods +:class:`~scrapy.throttler.ThrottlerScopeManager` and override only the methods whose behavior they want to change; implementing the -:class:`~scrapy.throttling.ThrottlingScopeManagerProtocol` from scratch is also +:class:`~scrapy.throttler.ThrottlerScopeManagerProtocol` from scratch is also supported. For example, this manager disables exponential :ref:`backoff `, so a scope relies solely on its configured delay and quota: .. code-block:: python :caption: ``myproject/throttling.py`` - from scrapy.throttling import ThrottlingScopeManager + from scrapy.throttler import ThrottlerScopeManager - class FixedWindowScopeManager(ThrottlingScopeManager): + class FixedWindowScopeManager(ThrottlerScopeManager): def record_backoff(self, *args, **kwargs): pass # never back off -.. _throttling-aware-scheduler: +.. _throttler-aware-scheduler: Throttling-aware scheduling =========================== By default, throttling is enforced at the engine, where a request waiting on -its :ref:`throttling scopes ` holds a concurrency slot. In a +its :ref:`throttler scopes ` holds a concurrency slot. In a crawl that mixes heavily-throttled scopes with unthrottled ones, this can let throttled requests starve unthrottled ones that could be sent right away (**head-of-line blocking**; Scrapy logs a warning the first time throttled requests start consuming the global concurrency budget while they wait). -:class:`~scrapy.core.scheduler.ThrottlingAwareScheduler` avoids this. To enable +:class:`~scrapy.core.scheduler.ThrottlerAwareScheduler` avoids this. To enable it: .. code-block:: python :caption: ``settings.py`` - SCHEDULER = "scrapy.core.scheduler.ThrottlingAwareScheduler" - SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ThrottlingAwarePriorityQueue" + SCHEDULER = "scrapy.core.scheduler.ThrottlerAwareScheduler" + SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ThrottlerAwarePriorityQueue" -.. autoclass:: scrapy.core.scheduler.ThrottlingAwareScheduler +.. autoclass:: scrapy.core.scheduler.ThrottlerAwareScheduler .. _throttling-examples: @@ -587,14 +587,14 @@ Examples Alternative domain throttling ----------------------------- -If you are not happy with the :ref:`default throttling scope behavior +If you are not happy with the :ref:`default throttler scope behavior ` with regards to domains and subdomains, you can change it. Alternative approaches include: -- Using the **highest-level registrable domain** as the throttling scope, +- Using the **highest-level registrable domain** as the throttler scope, e.g. https://books.toscrape.com and https://toscrape.com both get a - ``toscrape.com`` throttling scope. + ``toscrape.com`` throttler scope. This allows to apply the same throttling settings to all subdomains of a registrable domain. @@ -605,24 +605,24 @@ Alternative approaches include: :caption: ``settings.py`` import tldextract - from scrapy.throttling import ThrottlingManager, scope_cache + from scrapy.throttler import Throttler, scope_cache from scrapy.utils.httpobj import urlparse_cached - class MyThrottlingManager(ThrottlingManager): + class MyThrottler(Throttler): @scope_cache async def get_scopes(self, request): extracted = tldextract.extract(request.url) return extracted.registered_domain or urlparse_cached(request).netloc - THROTTLING_MANAGER = MyThrottlingManager + THROTTLER = MyThrottler -- Using **multiple throttling scopes per request**, one per registrable +- Using **multiple throttler scopes per request**, one per registrable domain and for every higher-level subdomain, e.g. https://books.toscrape.com and https://toscrape.com both get a - ``toscrape.com`` throttling scope, but https://books.toscrape.com also - gets a ``books.toscrape.com`` throttling scope. + ``toscrape.com`` throttler scope, but https://books.toscrape.com also + gets a ``books.toscrape.com`` throttler scope. This allows to apply the same throttling settings to all subdomains of a registrable domain, but also allows applying further restrictions on each @@ -634,11 +634,11 @@ Alternative approaches include: :caption: ``settings.py`` import tldextract - from scrapy.throttling import ThrottlingManager, scope_cache + from scrapy.throttler import Throttler, scope_cache from scrapy.utils.httpobj import urlparse_cached - class MyThrottlingManager(ThrottlingManager): + class MyThrottler(Throttler): @scope_cache async def get_scopes(self, request): extracted = tldextract.extract(request.url) @@ -648,8 +648,8 @@ Alternative approaches include: return {extracted.registered_domain, extracted.fqdn} - THROTTLING_MANAGER = MyThrottlingManager - THROTTLING_SCOPES = { + THROTTLER = MyThrottler + THROTTLER_SCOPES = { "toscrape.com": {"concurrency": 32}, "books.toscrape.com": {"concurrency": 24}, "quotes.toscrape.com": {"concurrency": 16}, @@ -669,16 +669,16 @@ To apply different throttling settings to different endpoints of the same domain and not enforce any common throttling, effectively treating them as different domains: -- Implement a :ref:`throttling manager ` that sets - endpoint-specific throttling scopes for that domain: +- Implement a :ref:`throttler ` that sets + endpoint-specific throttler scopes for that domain: .. code-block:: python - from scrapy.throttling import ThrottlingManager, scope_cache + from scrapy.throttler import Throttler, scope_cache from scrapy.utils.httpobj import urlparse_cached - class MyThrottlingManager(ThrottlingManager): + class MyThrottler(Throttler): @scope_cache async def get_scopes(self, request): parsed_url = urlparse_cached(request) @@ -686,13 +686,13 @@ different domains: return await super().get_scopes(request) return f"{parsed_url.netloc}{parsed_url.path}" -- Use the :setting:`THROTTLING_SCOPES` setting to set different throttling +- Use the :setting:`THROTTLER_SCOPES` setting to set different throttling settings per endpoint: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "api.toscrape.com/fast-endpoint": {"concurrency": 1000, "delay": 0.08}, "api.toscrape.com/slow-endpoint": {"delay": 5.0}, } @@ -707,35 +707,35 @@ Imagine you are sending requests to a web scraping API, e.g. to avoid bans. Unless that API provides a Scrapy plugin to make it easier to use, you may want to: -- Use the :setting:`THROTTLING_SCOPES` setting to increase concurrency for +- Use the :setting:`THROTTLER_SCOPES` setting to increase concurrency for API requests. For example: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "api.toscrape.com": {"concurrency": 1000, "delay": 0.08}, } -- Implement a :ref:`throttling manager ` that: +- Implement a :ref:`throttler ` that: - - Adds a throttling scope for the URL being scraped. + - Adds a throttler scope for the URL being scraped. For example, if you request ``https://api.toscrape.com/?url=https://example.com``, by default it - will get a ``api.toscrape.com`` throttling scope, but it should also - get the ``example.com`` throttling scope: + will get a ``api.toscrape.com`` throttler scope, but it should also + get the ``example.com`` throttler scope: .. code-block:: python from urllib.parse import urlparse - from scrapy.throttling import add_scope, ThrottlingManager, scope_cache + from scrapy.throttler import add_scope, Throttler, scope_cache from scrapy.utils.httpobj import urlparse_cached from w3lib.url import url_query_parameter - class MyThrottlingManager(ThrottlingManager): + class MyThrottler(Throttler): @scope_cache async def get_scopes(self, request): scopes = await super().get_scopes(request) @@ -756,7 +756,7 @@ to: .. code-block:: python - from scrapy.throttling import iter_scopes + from scrapy.throttler import iter_scopes from scrapy.utils.httpobj import urlparse_cached @@ -795,21 +795,21 @@ Cost-capped throttling Imagine you are using an API that charges different requests differently, e.g. based on the features used, and you want to limit how much you spend per time -window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas -` for that: +window (:setting:`THROTTLER_WINDOW`). You can use :ref:`throttler quotas +` for that: -- Implement a :ref:`throttling manager ` that: +- Implement a :ref:`throttler ` that: - - Sets a ``cost`` throttling scope on each request to some estimation + - Sets a ``cost`` throttler scope on each request to some estimation based e.g. on request URL parameters: .. code-block:: python from scrapy.utils.httpobj import urlparse_cached - from scrapy.throttling import ThrottlingManager, scope_cache + from scrapy.throttler import Throttler, scope_cache - class MyThrottlingManager(ThrottlingManager): + class MyThrottler(Throttler): @scope_cache async def get_scopes(self, request): scopes = await super().get_scopes(request) @@ -840,13 +840,13 @@ window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas self.throttler.reconcile_quota("cost", consumed=actual - estimated) return response -- Use the :setting:`THROTTLING_SCOPES` setting to set a maximum cost per time +- Use the :setting:`THROTTLER_SCOPES` setting to set a maximum cost per time window: .. code-block:: python :caption: ``settings.py`` - THROTTLING_SCOPES = { + THROTTLER_SCOPES = { "cost": {"quota": 100.0}, } @@ -858,24 +858,24 @@ window (:setting:`THROTTLING_WINDOW`). You can use :ref:`throttling quotas Per-IP concurrency limiting --------------------------- -A concurrency limit keyed by IP is just a throttling scope whose id is the +A concurrency limit keyed by IP is just a throttler scope whose id is the request's IP, with a ``concurrency`` limit. A request then carries two scopes, its domain and its IP, and is only sent when **both** allow it (see -:ref:`multiple-throttling-scopes`). +:ref:`multiple-throttler-scopes`). -- Implement a :ref:`throttling manager ` that adds +- Implement a :ref:`throttler ` that adds the request's IP as a second scope: .. code-block:: python import socket - from scrapy.throttling import ThrottlingManager, add_scope, scope_cache + from scrapy.throttler import Throttler, add_scope, scope_cache from scrapy.utils.asyncio import run_in_thread from scrapy.utils.httpobj import urlparse_cached - class IPThrottlingManager(ThrottlingManager): + class IPThrottler(Throttler): @scope_cache async def get_scopes(self, request): scopes = await super().get_scopes(request) @@ -887,6 +887,7 @@ its domain and its IP, and is only sent when **both** allow it (see Additional settings =================== + - .. setting:: BACKOFF_ENABLED :setting:`BACKOFF_ENABLED` (default: ``True``) @@ -933,7 +934,7 @@ Additional settings :setting:`BACKOFF_WINDOW` (default: ``60.0``) Time window, in seconds, used by :ref:`backoff `. A - :ref:`throttling scope ` must go this many seconds + :ref:`throttler scope ` must go this many seconds without a new backoff trigger (an HTTP error code from :setting:`BACKOFF_HTTP_CODES` or an exception from :setting:`BACKOFF_EXCEPTIONS`) before its delay decreases @@ -954,18 +955,18 @@ Additional settings If ``True``, ``0.5`` (i.e. ±50%) is used as the randomization factor. If ``False``, no randomization is applied. -- .. setting:: THROTTLING_DEBUG +- .. setting:: THROTTLER_DEBUG - :setting:`THROTTLING_DEBUG` (default: ``False``) + :setting:`THROTTLER_DEBUG` (default: ``False``) Whether to log :ref:`throttling ` decisions (per-scope delays, backoff steps and recoveries) for debugging. -- .. setting:: THROTTLING_SCOPE_LIMIT +- .. setting:: THROTTLER_SCOPE_LIMIT - :setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``) + :setting:`THROTTLER_SCOPE_LIMIT` (default: ``100000``) - Maximum number of :ref:`throttling scope ` states kept + Maximum number of :ref:`throttler scope ` states kept in memory at once, to bound memory usage on broad crawls that touch a large number of scopes (e.g. domains). @@ -975,15 +976,15 @@ Additional settings evicted, so the limit may be temporarily exceeded if that many scopes are busy at once. Set to ``0`` to disable the limit. - This complements :setting:`THROTTLING_SCOPE_MAX_IDLE`, which evicts scopes + This complements :setting:`THROTTLER_SCOPE_MAX_IDLE`, which evicts scopes by inactivity time rather than by count. -- .. setting:: THROTTLING_SCOPE_MAX_IDLE +- .. setting:: THROTTLER_SCOPE_MAX_IDLE - :setting:`THROTTLING_SCOPE_MAX_IDLE` (default: ``3600.0``) + :setting:`THROTTLER_SCOPE_MAX_IDLE` (default: ``3600.0``) - Seconds of inactivity after which the state of a :ref:`throttling scope - ` is evicted from memory to bound memory usage on + Seconds of inactivity after which the state of a :ref:`throttler scope + ` is evicted from memory to bound memory usage on long-running crawls. Set to ``0`` to never evict. Scopes in active backoff are never evicted. @@ -992,24 +993,24 @@ Additional settings API === -.. autoclass:: scrapy.throttling.ThrottlingManagerProtocol +.. autoclass:: scrapy.throttler.ThrottlerProtocol :members: :member-order: bysource -.. autoclass:: scrapy.throttling.ThrottlingManager +.. autoclass:: scrapy.throttler.Throttler -.. autoclass:: scrapy.throttling.ThrottlingScopeManagerProtocol +.. autoclass:: scrapy.throttler.ThrottlerScopeManagerProtocol :members: :member-order: bysource -.. autoclass:: scrapy.throttling.ThrottlingScopeManager +.. autoclass:: scrapy.throttler.ThrottlerScopeManager -.. autoclass:: scrapy.pqueues.ThrottlingAwarePriorityQueue +.. autoclass:: scrapy.pqueues.ThrottlerAwarePriorityQueue -.. autoclass:: scrapy.throttling.ThrottlingScopeConfig +.. autoclass:: scrapy.throttler.ThrottlerScopeConfig -.. autoclass:: scrapy.throttling.BackoffConfig +.. autoclass:: scrapy.throttler.BackoffConfig -.. autofunction:: scrapy.throttling.scope_cache -.. autofunction:: scrapy.throttling.add_scope -.. autofunction:: scrapy.throttling.iter_scopes +.. autofunction:: scrapy.throttler.scope_cache +.. autofunction:: scrapy.throttler.add_scope +.. autofunction:: scrapy.throttler.iter_scopes diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 6a3c155d5..ca3162bc1 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -28,10 +28,7 @@ if TYPE_CHECKING: from scrapy.http import Response from scrapy.settings import BaseSettings from scrapy.signalmanager import SignalManager - from scrapy.throttling import ( - ThrottlingManagerProtocol, - ThrottlingScopeManagerProtocol, - ) + from scrapy.throttler import ThrottlerProtocol, ThrottlerScopeManagerProtocol from scrapy.utils.asyncio import CallLaterResult @@ -92,7 +89,7 @@ class _DeprecatedSlotView: self, downloader: Downloader, key: str, - scope: ThrottlingScopeManagerProtocol, + scope: ThrottlerScopeManagerProtocol, ) -> None: self._downloader = downloader self._key = key @@ -114,10 +111,10 @@ class _DeprecatedSlotView: if r.meta.get(Downloader.DOWNLOAD_SLOT) == self._key } - # This deprecated view reads throttling scope state from private attributes + # This deprecated view reads throttler scope state from private attributes # of the default scope manager rather than through the scope manager # protocol: these are read-only compatibility accessors, so keeping them off - # the protocol avoids forcing custom THROTTLING_SCOPE_MANAGER implementations + # the protocol avoids forcing custom THROTTLER_SCOPE_MANAGER implementations # to provide members that only exist to feed this shim. A custom manager that # lacks the attribute simply falls back to the historical default. @property @@ -168,9 +165,7 @@ class _DeprecatedSlotsView(Mapping[str, _DeprecatedSlotView]): __slots__ = ("_downloader", "_throttler") - def __init__( - self, downloader: Downloader, throttler: ThrottlingManagerProtocol - ) -> None: + def __init__(self, downloader: Downloader, throttler: ThrottlerProtocol) -> None: self._downloader = downloader self._throttler = throttler @@ -216,7 +211,7 @@ class Downloader: ) if self.per_slot_settings: warnings.warn( - "The DOWNLOAD_SLOTS setting is deprecated. Use THROTTLING_SCOPES for " + "The DOWNLOAD_SLOTS setting is deprecated. Use THROTTLER_SCOPES for " "per-domain configuration instead.", category=ScrapyDeprecationWarning, stacklevel=2, @@ -264,7 +259,7 @@ class Downloader: @property def slots(self) -> _DeprecatedSlotsView: warnings.warn( - "Downloader.slots is deprecated. Use the throttling manager API instead.", + "Downloader.slots is deprecated. Use the throttler API instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py index 66308389a..b0ddca604 100644 --- a/scrapy/core/downloader/handlers/_base_http.py +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -17,7 +17,7 @@ class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): def _max_per_host_concurrency(settings: BaseSettings) -> int: """Highest per-host concurrency the throttler may admit: the per-domain limit, the default ``other``-scope limit, and any explicit - :setting:`THROTTLING_SCOPES` concurrency. + :setting:`THROTTLER_SCOPES` concurrency. Since :setting:`CONCURRENT_REQUESTS` caps the total number of requests in flight, no host can ever exceed it, so it is also the upper bound of the @@ -26,11 +26,11 @@ class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): global_concurrency = settings.getint("CONCURRENT_REQUESTS") candidates = [ settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN"), - settings.getint("THROTTLING_SCOPE_CONCURRENCY"), + settings.getint("THROTTLER_SCOPE_CONCURRENCY"), ] candidates += [ int(scope["concurrency"]) - for scope in settings.getdict("THROTTLING_SCOPES").values() + for scope in settings.getdict("THROTTLER_SCOPES").values() if "concurrency" in scope ] return min(max(candidates), global_concurrency) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2dc64c5c3..34acd5051 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -131,9 +131,9 @@ class ExecutionEngine: self._start_request_processing_awaitable: ( asyncio.Future[None] | Deferred[None] | None ) = None - # Requests currently held by the throttling manager, waiting for their + # Requests currently held by the throttler, waiting for their # scopes to allow them through to the downloader. - self._throttling_waiting: set[Request] = set() + self._throttler_waiting: set[Request] = set() # Number of in-flight asynchronous enqueue operations (see # ``_enqueue_request_async``), so the spider is not considered idle # while a request is still on its way into the scheduler. @@ -149,7 +149,7 @@ class ExecutionEngine: # Whether the scheduler exposes enqueue_request_async, cached once per # crawl in ``open_spider_async``; checked for every scheduled request. self._scheduler_enqueues_async: bool = False - self._throttling_backout_warned: bool = False + self._throttler_backout_warned: bool = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -412,40 +412,40 @@ class ExecutionEngine: or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() - or self._throttling_needs_backout() + or self._throttler_needs_backout() ) - def _throttling_needs_backout(self) -> bool: - """Apply backpressure while requests are held by the throttling manager. + def _throttler_needs_backout(self) -> bool: + """Apply backpressure while requests are held by the throttler. Requests waiting on throttling do not occupy a downloader concurrency slot, so without this check the engine would keep draining the scheduler into pending throttling waits. Count them together with the in-flight requests against the global concurrency limit. """ - if not self._throttling_waiting: + if not self._throttler_waiting: return False if ( - len(self._throttling_waiting) + len(self.downloader.active) + len(self._throttler_waiting) + len(self.downloader.active) < self.downloader.total_concurrency ): return False - self._maybe_warn_throttling_backout() + self._maybe_warn_throttler_backout() return True - def _maybe_warn_throttling_backout(self) -> None: - if self._throttling_backout_warned: + def _maybe_warn_throttler_backout(self) -> None: + if self._throttler_backout_warned: return - # A throttling-aware scheduler holds time-blocked requests itself instead - # of letting them pile up in _throttling_waiting and back-pressure the + # A throttler-aware scheduler holds time-blocked requests itself instead + # of letting them pile up in _throttler_waiting and back-pressure the # engine here, so the warning does not apply when one is in use. if self._get_next_request_delay is not None: return - self._throttling_backout_warned = True + self._throttler_backout_warned = True logger.warning( "Throttling is holding requests back and they are now consuming the " "global concurrency budget while they wait for a free slot. Consider " - "switching to scrapy.core.scheduler.ThrottlingAwareScheduler, which " + "switching to scrapy.core.scheduler.ThrottlerAwareScheduler, which " "holds throttled requests without consuming concurrency slots.", extra={"spider": self.spider}, ) @@ -523,7 +523,7 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False - if self._throttling_waiting: # requests held by the throttling manager + if self._throttler_waiting: # requests held by the throttler return False if self._scheduling: # requests still on their way into the scheduler return False @@ -612,18 +612,18 @@ class ExecutionEngine: return response_or_request @inlineCallbacks - def _acquire_throttling( + def _acquire_throttler( self, request: Request ) -> Generator[Deferred[Any], Any, None]: """Wait at the throttling gate before *request* is sent, tracking it as held meanwhile.""" - self._throttling_waiting.add(request) + self._throttler_waiting.add(request) throttler = self.crawler.throttler assert throttler is not None try: yield deferred_from_coro(throttler.acquire(request)) finally: - self._throttling_waiting.discard(request) + self._throttler_waiting.discard(request) @inlineCallbacks def _download( @@ -634,12 +634,12 @@ class ExecutionEngine: throttler = self.crawler.throttler assert throttler is not None - # A throttling-aware scheduler reserves the request (a concurrency slot) + # A throttler-aware scheduler reserves the request (a concurrency slot) # before handing it here, so everything past this point runs inside the # try/finally that releases it, even if add_request() were to raise. try: self._slot.add_request(request) - yield self._acquire_throttling(request) + yield self._acquire_throttler(request) result: Response | Request if self._downloader_fetch_needs_spider: result = yield self.downloader.fetch(request, self.spider) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index fcdb67cab..3ffb6a78c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, cast from twisted.internet.defer import Deferred # noqa: TC002 from scrapy.spiders import Spider # noqa: TC001 -from scrapy.throttling import iter_scopes +from scrapy.throttler import iter_scopes from scrapy.utils.job import job_dir from scrapy.utils.misc import build_from_crawler, load_object @@ -26,7 +26,7 @@ if TYPE_CHECKING: from scrapy.http.request import Request from scrapy.pqueues import ScrapyPriorityQueue from scrapy.statscollectors import StatsCollector - from scrapy.throttling import ThrottlingManagerProtocol + from scrapy.throttler import ThrottlerProtocol logger = logging.getLogger(__name__) @@ -79,7 +79,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta): Asynchronous counterpart of :meth:`enqueue_request`, following the same return contract. When a scheduler defines it, the engine awaits it *instead of* calling :meth:`enqueue_request`. Define it when enqueuing a - request needs to ``await`` (e.g. to resolve throttling scopes + request needs to ``await`` (e.g. to resolve throttler scopes asynchronously). .. method:: get_next_request_delay() @@ -238,7 +238,7 @@ class Scheduler(BaseScheduler): ------------------------- While pending requests are below the configured values of - :setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLING_SCOPE_CONCURRENCY`, + :setting:`CONCURRENT_REQUESTS` or :setting:`THROTTLER_SCOPE_CONCURRENCY`, those requests are sent concurrently. As a result, the first few requests of a crawl may not follow the desired @@ -410,7 +410,7 @@ class Scheduler(BaseScheduler): Extra positional arguments are forwarded to the underlying queue ``push`` calls; subclasses that store requests under additional keys - (e.g. a throttling scope set) pass them through here. + (e.g. a throttler scope set) pass them through here. """ assert self.stats is not None if self._dqpush(request, *push_args): @@ -535,14 +535,14 @@ class Scheduler(BaseScheduler): json.dump(state, f) -class ThrottlingAwareScheduler(Scheduler): +class ThrottlerAwareScheduler(Scheduler): """A :setting:`SCHEDULER` that only ever hands the engine requests that - their :ref:`throttling scopes ` allow to be sent **right + their :ref:`throttler scopes ` allow to be sent **right now**. The default scheduler hands requests to the engine as concurrency allows and lets them wait at the throttling gate - (:meth:`~scrapy.throttling.ThrottlingManagerProtocol.acquire`) while holding + (:meth:`~scrapy.throttler.ThrottlerProtocol.acquire`) while holding a concurrency slot. When a crawl mixes heavily-throttled scopes with unthrottled ones, enough throttled requests waiting on a clock can fill :setting:`CONCURRENT_REQUESTS` and starve unthrottled requests that could be @@ -551,7 +551,7 @@ class ThrottlingAwareScheduler(Scheduler): (beyond what is needed to track each distinct scope set) and cannot starve other scopes. - It also honors the per-request :reqmeta:`throttling_delay`, holding an + It also honors the per-request :reqmeta:`delay`, holding an individual request back without blocking others that share its scopes, which the default scheduler cannot do. @@ -560,7 +560,7 @@ class ThrottlingAwareScheduler(Scheduler): broken by preferring the least-busy scopes. It requires :setting:`SCHEDULER_PRIORITY_QUEUE` to be set to - :class:`~scrapy.pqueues.ThrottlingAwarePriorityQueue` (or a compatible + :class:`~scrapy.pqueues.ThrottlerAwarePriorityQueue` (or a compatible subclass). """ @@ -569,18 +569,18 @@ class ThrottlingAwareScheduler(Scheduler): if not hasattr(self.mqs, "get_next_request_delay"): raise ValueError( f"{type(self).__name__} requires SCHEDULER_PRIORITY_QUEUE to be " - f"set to a throttling-aware priority queue such as " - f"scrapy.pqueues.ThrottlingAwarePriorityQueue, but the " + f"set to a throttler-aware priority queue such as " + f"scrapy.pqueues.ThrottlerAwarePriorityQueue, but the " f"configured one ({type(self.mqs).__name__}) is not." ) assert self.crawler is not None assert self.crawler.throttler is not None - self._throttler: ThrottlingManagerProtocol = self.crawler.throttler + self._throttler: ThrottlerProtocol = self.crawler.throttler return result def enqueue_request(self, request: Request) -> bool: raise RuntimeError( - "ThrottlingAwareScheduler requires the asynchronous enqueue path; " + "ThrottlerAwareScheduler requires the asynchronous enqueue path; " "enqueue_request_async() is used by the engine instead of " "enqueue_request()." ) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0198ef93e..508e6fabf 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -46,7 +46,7 @@ if TYPE_CHECKING: from scrapy.logformatter import LogFormatter from scrapy.statscollectors import StatsCollector - from scrapy.throttling import ThrottlingManagerProtocol + from scrapy.throttler import ThrottlerProtocol from scrapy.utils.request import RequestFingerprinterProtocol @@ -84,13 +84,13 @@ class Crawler: self.stats: StatsCollector | None = None self.logformatter: LogFormatter | None = None self.request_fingerprinter: RequestFingerprinterProtocol | None = None - self.throttler: ThrottlingManagerProtocol | None = None - """The throttling manager of this crawler, an instance of - :setting:`THROTTLING_MANAGER`. + self.throttler: ThrottlerProtocol | None = None + """The throttler of this crawler, an instance of + :setting:`THROTTLER`. It is ``None`` until the crawl starts. Components can use it to inspect or drive :ref:`throttling ` at run time, e.g. through - :meth:`~scrapy.throttling.ThrottlingManagerProtocol.delay_scope`.""" + :meth:`~scrapy.throttler.ThrottlerProtocol.delay_scope`.""" self.spider: Spider | None = None self.engine: ExecutionEngine | None = None @@ -106,7 +106,7 @@ class Crawler: self.addons.load_settings(self.settings) self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY") self._apply_deprecated_spider_attr( - "max_concurrent_requests", "THROTTLING_SCOPE_CONCURRENCY" + "max_concurrent_requests", "THROTTLER_SCOPE_CONCURRENCY" ) self.stats = load_object(self.settings["STATS_CLASS"])(self) @@ -118,7 +118,7 @@ class Crawler: self, ) self.throttler = build_from_crawler( - load_object(self.settings["THROTTLING_MANAGER"]), + load_object(self.settings["THROTTLER"]), self, ) @@ -185,7 +185,7 @@ class Crawler: return warnings.warn( f"The {attr!r} spider attribute is deprecated. Use the {setting} " - f"setting or THROTTLING_SCOPES instead.", + f"setting or THROTTLER_SCOPES instead.", category=ScrapyDeprecationWarning, stacklevel=3, ) diff --git a/scrapy/downloadermiddlewares/backoff.py b/scrapy/downloadermiddlewares/backoff.py index 80c9eb140..37a10ca21 100644 --- a/scrapy/downloadermiddlewares/backoff.py +++ b/scrapy/downloadermiddlewares/backoff.py @@ -4,7 +4,7 @@ import logging from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured -from scrapy.throttling import iter_scopes +from scrapy.throttler import iter_scopes from scrapy.utils._headers import _parse_ratelimit_reset, _parse_retry_after from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.misc import _load_objects @@ -16,7 +16,7 @@ if TYPE_CHECKING: import scrapy from scrapy.crawler import Crawler from scrapy.http import Request, Response - from scrapy.throttling import ThrottlingManagerProtocol + from scrapy.throttler import ThrottlerProtocol logger = logging.getLogger(__name__) @@ -28,9 +28,9 @@ class BackoffMiddleware: It observes every response and download exception and, for those matching :setting:`BACKOFF_HTTP_CODES` or :setting:`BACKOFF_EXCEPTIONS` (globally or - per :setting:`THROTTLING_SCOPES` scope), tells the :ref:`throttling manager + per :setting:`THROTTLER_SCOPES` scope), tells the :ref:`throttler ` to back off the request's scopes through its - :meth:`~scrapy.throttling.ThrottlingManagerProtocol.back_off` API. + :meth:`~scrapy.throttler.ThrottlerProtocol.back_off` API. It is enabled by default; set :setting:`BACKOFF_ENABLED` to ``False`` to disable it without removing it from :setting:`DOWNLOADER_MIDDLEWARES`. @@ -46,7 +46,7 @@ class BackoffMiddleware: if not crawler.settings.getbool("BACKOFF_ENABLED"): raise NotConfigured assert crawler.throttler is not None - self._throttler: ThrottlingManagerProtocol = crawler.throttler + self._throttler: ThrottlerProtocol = crawler.throttler settings = crawler.settings self._http_codes: set[int] = { int(code) for code in settings.getlist("BACKOFF_HTTP_CODES") @@ -61,7 +61,7 @@ class BackoffMiddleware: # pre-filter to skip outcomes that no scope backs off on. self._any_http_codes: set[int] = set(self._http_codes) self._any_exceptions: tuple[type[BaseException], ...] = self._exceptions - for scope_id, scope_config in settings.getdict("THROTTLING_SCOPES").items(): + for scope_id, scope_config in settings.getdict("THROTTLER_SCOPES").items(): backoff = scope_config.get("backoff") or {} if "http_codes" in backoff: codes = {int(code) for code in backoff["http_codes"]} diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 35b347d08..aa4f2cdab 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -184,7 +184,7 @@ class BaseRedirectMiddleware: """Delay the redirect when *response* carries a ``Retry-After`` header. The delay is capped at :setting:`REDIRECT_MAX_DELAY` and applied through - the :reqmeta:`throttling_delay` request metadata key, which holds back + the :reqmeta:`delay` request metadata key, which holds back this request only (without counting as a :ref:`backoff ` trigger for its scopes). :setting:`REDIRECT_MAX_DELAY` set to ``0`` disables it. @@ -194,11 +194,11 @@ class BaseRedirectMiddleware: retry_after = _parse_retry_after(response) if retry_after is None: return - redirect_request.meta["throttling_delay"] = min(retry_after, self.max_delay) + redirect_request.meta["delay"] = min(retry_after, self.max_delay) # This is a fresh request that may inherit an already-honored delay from # its source request's meta; clear that state so the new delay applies. - redirect_request.meta.pop("_throttling_delayed", None) - redirect_request.meta.pop("_throttling_delay_deadline", None) + redirect_request.meta.pop("_throttler_delayed", None) + redirect_request.meta.pop("_throttler_delay_deadline", None) def _redirect_request_using_get( self, request: Request, response: Response, redirect_url: str diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 9f3736c5b..befdfc293 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http import Response - from scrapy.throttling import ThrottlingManagerProtocol + from scrapy.throttler import ThrottlerProtocol logger = logging.getLogger(__name__) @@ -28,7 +28,7 @@ class AutoThrottle: warn( "You have set the AUTOTHROTTLE_ENABLED setting to True, however " - "the AutoThrottle extension is deprecated; use throttling and " + "the AutoThrottle extension is deprecated; use throttler and " "backoff settings instead: " "https://docs.scrapy.org/en/latest/topics/throttling.html", ScrapyDeprecationWarning, @@ -99,9 +99,7 @@ class AutoThrottle: extra={"spider": spider}, ) - def _scope_delay( - self, throttler: ThrottlingManagerProtocol, scope_id: str - ) -> float: + def _scope_delay(self, throttler: ThrottlerProtocol, scope_id: str) -> float: """Return the current delay of *scope_id*, applying AUTOTHROTTLE_START_DELAY the first time the scope is seen.""" delay = throttler.get_scope_delay(scope_id) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 25008cc17..520550874 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: from scrapy import Request from scrapy.crawler import Crawler - from scrapy.throttling import ScopeID, ThrottlingManagerProtocol + from scrapy.throttler import ScopeID, ThrottlerProtocol logger = logging.getLogger(__name__) @@ -329,7 +329,7 @@ class DownloaderAwarePriorityQueue: ) assert crawler.throttler is not None - self._throttler: ThrottlingManagerProtocol = crawler.throttler + self._throttler: ThrottlerProtocol = crawler.throttler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key @@ -441,10 +441,10 @@ def _scope_set_from_key(key: str) -> frozenset[ScopeID]: return frozenset(json.loads(key)) -class ThrottlingAwarePriorityQueue: +class ThrottlerAwarePriorityQueue: """Priority queue that only ever pops a request that can be sent right now - based on its :ref:`throttling scope set ` and - per-request :reqmeta:`throttling_delay`. + based on its :ref:`throttler scope set ` and + per-request :reqmeta:`delay`. The downstream queue class must support ``peek``. @@ -460,6 +460,22 @@ class ThrottlingAwarePriorityQueue: persistence directory, and inside it this class creates a subdirectory per scope set, named from a path-safe, order-independent encoding of its scope ids. + + For example, a request whose scope set is ``{"example.com", + "cost:group-1"}`` is stored under a subdirectory derived in two steps: + + #. The scope ids are sorted and JSON-encoded into an order-independent key + (so ``{"example.com", "cost:group-1"}`` and ``{"cost:group-1", + "example.com"}`` map to the same one):: + + ["cost:group-1", "example.com"] + + #. That key is made path-safe: every character outside ``[A-Za-z0-9-._]`` + becomes ``_`` (here the ``:`` and the JSON quotes, brackets and spaces; + the ``-`` and ``.`` are kept), and an MD5 suffix disambiguates keys that + collapse to the same path:: + + __cost_group-1____example.com__-fc6ba2aff8f421bf981b662d77739902 """ @classmethod @@ -491,7 +507,7 @@ class ThrottlingAwarePriorityQueue: ): if slot_startprios and not isinstance(slot_startprios, dict): raise ValueError( - "ThrottlingAwarePriorityQueue accepts ``slot_startprios`` as a " + "ThrottlerAwarePriorityQueue accepts ``slot_startprios`` as a " f"dict; {slot_startprios.__class__!r} instance is passed. Most " "likely, it means the state is created by an incompatible " "priority queue. Only a crawl started with the same priority " @@ -499,7 +515,7 @@ class ThrottlingAwarePriorityQueue: ) assert crawler.throttler is not None - self._throttler: ThrottlingManagerProtocol = crawler.throttler + self._throttler: ThrottlerProtocol = crawler.throttler self.downstream_queue_cls: type[QueueProtocol] = downstream_queue_cls self._start_queue_cls: type[QueueProtocol] | None = start_queue_cls self.key: str = key @@ -509,15 +525,26 @@ class ThrottlingAwarePriorityQueue: if slot_startprios: for set_key, startprios in slot_startprios.items(): scope_set = _scope_set_from_key(set_key) - self.pqueues[scope_set] = self.pqfactory(scope_set, startprios) + self.pqueues[scope_set] = self._pqfactory(scope_set, startprios) - # Min-heap of (deadline, seq, scope_set, request) for requests held - # back by a per-request throttling_delay; seq keeps ordering stable and - # avoids comparing requests when deadlines tie. + # Requests held back by their own per-request delay wait + # here instead of in their scope-set queue, so a not-yet-due request + # never sits at a queue head and blocks the other requests that share + # its scopes (the head-of-line blocking this scheduler exists to + # avoid). Once the delay elapses, _promote_ready() moves the request + # into its scope-set queue, where it competes normally. + # + # This is a min-heap of (deadline, seq, scope_set, request). heapq + # needs a total order and always compares entries to keep its invariant; + # seq (a monotonic counter) makes the (deadline, seq) prefix totally + # ordered, so a deadline tie is broken there and the scope_set (a + # frozenset, whose < is only the partial subset order) and the request + # (no ordering at all) are never compared. The order among tied + # deadlines is irrelevant beyond being deterministic. self._delayed: list[tuple[float, int, frozenset[ScopeID], Request]] = [] self._delayed_seq: int = 0 - def pqfactory( + def _pqfactory( self, scope_set: frozenset[ScopeID], startprios: Iterable[int] = () ) -> ScrapyPriorityQueue: return ScrapyPriorityQueue( @@ -542,7 +569,7 @@ class ThrottlingAwarePriorityQueue: def _push_to_queue(self, request: Request, scope_set: frozenset[ScopeID]) -> None: if scope_set not in self.pqueues: - self.pqueues[scope_set] = self.pqfactory(scope_set) + self.pqueues[scope_set] = self._pqfactory(scope_set) self.pqueues[scope_set].push(request) def _promote_ready(self, now: float) -> None: @@ -559,7 +586,7 @@ class ThrottlingAwarePriorityQueue: # mark it consumed: the request must not be delayed again, and on resume # it must not re-block its scope set on a stale, no-longer-meaningful # deadline. - request.meta["_throttling_delayed"] = True + request.meta["_throttler_delayed"] = True try: self._push_to_queue(request, scope_set) except ValueError as e: @@ -586,7 +613,7 @@ class ThrottlingAwarePriorityQueue: Among the sendable queues (those whose scope set can be sent right now), the one whose head has the highest request priority is chosen; ties are broken by ascending load (the maximum - :meth:`~scrapy.throttling.ThrottlingManagerProtocol.get_scope_load` over the + :meth:`~scrapy.throttler.ThrottlerProtocol.get_scope_load` over the scopes of the queue), i.e. by preferring the least-busy scopes. """ self._promote_ready(time.monotonic()) @@ -618,12 +645,6 @@ class ThrottlingAwarePriorityQueue: del self.pqueues[scope_set] return request - def peek(self) -> Request | None: - selected = self._select() - if selected is None: - return None - return selected[1].peek() - def get_next_request_delay(self) -> float | None: now = time.monotonic() self._promote_ready(now) @@ -639,7 +660,7 @@ class ThrottlingAwarePriorityQueue: continue if delay is None or head_delay < delay: delay = head_delay - # A request held back only by its own throttling_delay is not in any + # A request held back only by its own delay is not in any # scope-set queue, so factor in when the earliest one is due. if self._delayed: next_delayed = max(0.0, self._delayed[0][0] - now) @@ -662,6 +683,3 @@ class ThrottlingAwarePriorityQueue: def __len__(self) -> int: queued = sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 return queued + len(self._delayed) - - def __contains__(self, scope_set: frozenset[ScopeID]) -> bool: - return scope_set in self.pqueues diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 20859ae30..b4ffa36f3 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -217,16 +217,16 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", - "THROTTLING_DEBUG", - "THROTTLING_MANAGER", - "THROTTLING_ROBOTSTXT_MAX_DELAY", - "THROTTLING_ROBOTSTXT_OBEY", - "THROTTLING_SCOPES", - "THROTTLING_SCOPE_CONCURRENCY", - "THROTTLING_SCOPE_LIMIT", - "THROTTLING_SCOPE_MANAGER", - "THROTTLING_SCOPE_MAX_IDLE", - "THROTTLING_WINDOW", + "THROTTLER", + "THROTTLER_DEBUG", + "THROTTLER_ROBOTSTXT_MAX_DELAY", + "THROTTLER_ROBOTSTXT_OBEY", + "THROTTLER_SCOPES", + "THROTTLER_SCOPE_CONCURRENCY", + "THROTTLER_SCOPE_LIMIT", + "THROTTLER_SCOPE_MANAGER", + "THROTTLER_SCOPE_MAX_IDLE", + "THROTTLER_WINDOW", "TWISTED_DNS_RESOLVER", "TWISTED_REACTOR", "TWISTED_REACTOR_ENABLED", @@ -607,16 +607,16 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) -THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager" -THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager" -THROTTLING_SCOPES = {} -THROTTLING_WINDOW = 60.0 -THROTTLING_ROBOTSTXT_OBEY = True -THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 -THROTTLING_SCOPE_CONCURRENCY = 1 -THROTTLING_SCOPE_LIMIT = 100000 -THROTTLING_SCOPE_MAX_IDLE = 3600.0 -THROTTLING_DEBUG = False +THROTTLER = "scrapy.throttler.Throttler" +THROTTLER_SCOPE_MANAGER = "scrapy.throttler.ThrottlerScopeManager" +THROTTLER_SCOPES = {} +THROTTLER_WINDOW = 60.0 +THROTTLER_ROBOTSTXT_OBEY = True +THROTTLER_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLER_SCOPE_CONCURRENCY = 1 +THROTTLER_SCOPE_LIMIT = 100000 +THROTTLER_SCOPE_MAX_IDLE = 3600.0 +THROTTLER_DEBUG = False TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index b010a7860..aa3de5312 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -21,9 +21,8 @@ ADDONS = {} # Obey robots.txt rules ROBOTSTXT_OBEY = True -# Concurrency and throttling settings #CONCURRENT_REQUESTS = 16 -THROTTLING_SCOPE_CONCURRENCY = 1 +THROTTLER_SCOPE_CONCURRENCY = 1 DOWNLOAD_DELAY = 1 # Disable cookies (enabled by default) diff --git a/scrapy/throttling.py b/scrapy/throttler.py similarity index 88% rename from scrapy/throttling.py rename to scrapy/throttler.py index 6785d607e..171f550ce 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttler.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) class BackoffConfig(TypedDict, total=False): """Per-scope override of the backoff settings. - Used as the value of the ``"backoff"`` key of :class:`ThrottlingScopeConfig` + Used as the value of the ``"backoff"`` key of :class:`ThrottlerScopeConfig` entries. Any key left out falls back to the corresponding global ``BACKOFF_*`` setting. """ @@ -46,8 +46,8 @@ class BackoffConfig(TypedDict, total=False): jitter: float | list[float] -class ThrottlingScopeConfig(TypedDict, total=False): - """Accepted keys of :setting:`THROTTLING_SCOPES` entries. +class ThrottlerScopeConfig(TypedDict, total=False): + """Accepted keys of :setting:`THROTTLER_SCOPES` entries. Every key is optional; missing keys fall back to the matching global setting (e.g. ``delay`` falls back to :setting:`DOWNLOAD_DELAY`). @@ -66,7 +66,7 @@ class ThrottlingScopeConfig(TypedDict, total=False): window: float manager: str | type - """Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for + """Import path or class of a custom :setting:`THROTTLER_SCOPE_MANAGER` for this scope.""" backoff: BackoffConfig @@ -83,8 +83,8 @@ RequestScopes = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, float | None] def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: """Iterate over the scope IDs of *scopes*, whatever its form. - :class:`~ThrottlingManagerProtocol.get_scopes` (and - :meth:`~ThrottlingManagerProtocol.get_resolved_scopes`) may return a single + :class:`~ThrottlerProtocol.get_scopes` (and + :meth:`~ThrottlerProtocol.get_resolved_scopes`) may return a single scope ID, an iterable of them, a ``{scope_id: quota}`` mapping, or ``None``; this helper normalizes any of those into an iterable of scope IDs, e.g. to react to a request's scopes in a custom middleware. @@ -95,7 +95,7 @@ def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]: def iter_scope_values(scopes: RequestScopes) -> Iterable[tuple[ScopeID, float | None]]: """Iterate over *scopes* as ``(scope_id, value)`` pairs. - For dict scopes the value is the expected :ref:`throttling quota + For dict scopes the value is the expected :ref:`throttler quota ` consumption; for every other form the value is ``None``. """ @@ -120,42 +120,42 @@ def _effective_priority(settings: BaseSettings, name: str) -> int: def _default_scope_concurrency(settings: BaseSettings) -> int: - """Return the default concurrency of a throttling scope that does not set + """Return the default concurrency of a throttler scope that does not set its own ``concurrency``. - This is :setting:`THROTTLING_SCOPE_CONCURRENCY`, except that the deprecated + This is :setting:`THROTTLER_SCOPE_CONCURRENCY`, except that the deprecated :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is bridged in when set at a higher :ref:`priority `. When neither is set explicitly, the historical :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` value is kept for backward compatibility (its default flips to - :setting:`THROTTLING_SCOPE_CONCURRENCY` in a future version; see + :setting:`THROTTLER_SCOPE_CONCURRENCY` in a future version; see :func:`_warn_on_deprecated_concurrency`). """ default_priority = SETTINGS_PRIORITIES["default"] domain_priority = _effective_priority(settings, "CONCURRENT_REQUESTS_PER_DOMAIN") - scope_priority = _effective_priority(settings, "THROTTLING_SCOPE_CONCURRENCY") + scope_priority = _effective_priority(settings, "THROTTLER_SCOPE_CONCURRENCY") if domain_priority > scope_priority: return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") if scope_priority > domain_priority: - return settings.getint("THROTTLING_SCOPE_CONCURRENCY") + return settings.getint("THROTTLER_SCOPE_CONCURRENCY") # Equal priority: on an explicit (higher-than-default) tie the new setting # wins; when neither is set (both at "default") keep the historical # per-domain value so existing behavior is preserved. if domain_priority <= default_priority: return settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") - return settings.getint("THROTTLING_SCOPE_CONCURRENCY") + return settings.getint("THROTTLER_SCOPE_CONCURRENCY") def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: """Warn about the concurrency settings bridged by :func:`_default_scope_concurrency`. Call once per crawl (see - :meth:`ThrottlingManager.__init__`). + :meth:`Throttler.__init__`). When :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is set explicitly, warn that it is deprecated. When neither concurrency setting is set explicitly, warn that the effective per-scope concurrency is still the deprecated setting's value (kept for backward compatibility) and will drop to - :setting:`THROTTLING_SCOPE_CONCURRENCY`'s default once the deprecated + :setting:`THROTTLER_SCOPE_CONCURRENCY`'s default once the deprecated setting is removed, so users can pin it explicitly.""" default_priority = SETTINGS_PRIORITIES["default"] domain_set = ( @@ -163,12 +163,12 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: > default_priority ) scope_set = ( - _effective_priority(settings, "THROTTLING_SCOPE_CONCURRENCY") > default_priority + _effective_priority(settings, "THROTTLER_SCOPE_CONCURRENCY") > default_priority ) if domain_set: warnings.warn( "The CONCURRENT_REQUESTS_PER_DOMAIN setting is deprecated, use " - "THROTTLING_SCOPE_CONCURRENCY instead.", + "THROTTLER_SCOPE_CONCURRENCY instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) @@ -178,14 +178,14 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: # guarded by test_deprecated_concurrency_defaults_differ rather than at # run time, so a crawl is never aborted over it. current = settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") - future = settings.getint("THROTTLING_SCOPE_CONCURRENCY") + future = settings.getint("THROTTLER_SCOPE_CONCURRENCY") warnings.warn( f"The effective per-scope (per-domain) concurrency is {current}, " f"the default of the deprecated CONCURRENT_REQUESTS_PER_DOMAIN " f"setting, which is still respected for backward compatibility. " f"Once CONCURRENT_REQUESTS_PER_DOMAIN is removed, it will drop to " - f"{future}, the default of THROTTLING_SCOPE_CONCURRENCY. Set " - f"THROTTLING_SCOPE_CONCURRENCY explicitly to choose a value and " + f"{future}, the default of THROTTLER_SCOPE_CONCURRENCY. Set " + f"THROTTLER_SCOPE_CONCURRENCY explicitly to choose a value and " f"silence this warning.", category=ScrapyDeprecationWarning, stacklevel=2, @@ -195,7 +195,7 @@ def _warn_on_deprecated_concurrency(settings: BaseSettings) -> None: def _warn_on_unachievable_concurrency(settings: BaseSettings) -> None: """Warn about configured concurrency limits that exceed :setting:`CONCURRENT_REQUESTS`. Call once per crawl (see - :meth:`ThrottlingManager.__init__`). + :meth:`Throttler.__init__`). :setting:`CONCURRENT_REQUESTS` caps the total number of requests in flight, so a per-scope (or per-domain) concurrency limit above it can never be @@ -204,12 +204,12 @@ def _warn_on_unachievable_concurrency(settings: BaseSettings) -> None: global_concurrency = settings.getint("CONCURRENT_REQUESTS") offenders: list[str] = [ f"{name}={settings.getint(name)}" - for name in ("CONCURRENT_REQUESTS_PER_DOMAIN", "THROTTLING_SCOPE_CONCURRENCY") + for name in ("CONCURRENT_REQUESTS_PER_DOMAIN", "THROTTLER_SCOPE_CONCURRENCY") if settings.getint(name) > global_concurrency ] offenders += [ - f"THROTTLING_SCOPES[{scope_id!r}]['concurrency']={config['concurrency']}" - for scope_id, config in settings.getdict("THROTTLING_SCOPES").items() + f"THROTTLER_SCOPES[{scope_id!r}]['concurrency']={config['concurrency']}" + for scope_id, config in settings.getdict("THROTTLER_SCOPES").items() if config.get("concurrency") is not None and int(config["concurrency"]) > global_concurrency ] @@ -249,8 +249,8 @@ def add_scope( dict. This is a utility function to help extending the output of - :meth:`~ThrottlingManagerProtocol.get_scopes`, e.g. in - :class:`ThrottlingManager` subclasses. + :meth:`~ThrottlerProtocol.get_scopes`, e.g. in + :class:`Throttler` subclasses. Adding a scope with a *value* fails if it is already present, so an existing :ref:`quota ` is never silently overwritten; adding it @@ -266,21 +266,21 @@ def add_scope( return result -class ThrottlingManagerProtocol(Protocol): - """A protocol for :setting:`THROTTLING_MANAGER` :ref:`components +class ThrottlerProtocol(Protocol): + """A protocol for :setting:`THROTTLER` :ref:`components `.""" async def get_scopes(self, request: Request) -> RequestScopes: - """Return the :ref:`throttling scopes ` that apply + """Return the :ref:`throttler scopes ` that apply to *request*. Return ``None`` if no scopes apply, a string for a single scope, an iterable of strings for multiple scopes, or a dict with scope names as - keys and :ref:`throttling quotas ` as values. + keys and :ref:`throttler quotas ` as values. """ def get_resolved_scopes(self, request: Request) -> RequestScopes: - """Return the :ref:`throttling scopes ` under which + """Return the :ref:`throttler scopes ` under which *request* was (or will be) sent, without re-resolving them. This is the synchronous counterpart of :meth:`get_scopes`: it returns @@ -314,8 +314,8 @@ class ThrottlingManagerProtocol(Protocol): *and* a concurrency slot is free in every scope. This is the synchronous, non-blocking counterpart of :meth:`acquire`, - used by a :ref:`throttling-aware scheduler - ` to decide whether a request can be + used by a :ref:`throttler-aware scheduler + ` to decide whether a request can be dequeued now. It assumes the scopes of *request* have already been resolved (e.g. by an earlier :meth:`get_scopes` call at enqueue time). """ @@ -325,7 +325,7 @@ class ThrottlingManagerProtocol(Protocol): scopes and mark *request* as reserved, so that a later :meth:`acquire` for it returns immediately without reserving again. - A :ref:`throttling-aware scheduler ` calls + A :ref:`throttler-aware scheduler ` calls this when it decides to dequeue *request* (after :meth:`is_ready` returned ``True``). The reservation is released by :meth:`release`. """ @@ -335,8 +335,8 @@ class ThrottlingManagerProtocol(Protocol): *request* would be open, or ``None`` if no time-based gate is currently blocking it (only a concurrency slot could be). - Used by a :ref:`throttling-aware scheduler - ` to schedule a wakeup when all pending + Used by a :ref:`throttler-aware scheduler + ` to schedule a wakeup when all pending requests are time-blocked. """ @@ -354,19 +354,19 @@ class ThrottlingManagerProtocol(Protocol): active sends divided by its concurrency limit (or by the global :setting:`CONCURRENT_REQUESTS` when the scope has no explicit limit). - Used by a :ref:`throttling-aware scheduler - ` to balance dequeuing across scopes, + Used by a :ref:`throttler-aware scheduler + ` to balance dequeuing across scopes, preferring the least-loaded ones. """ def get_request_delay(self, request: Request, now: float | None = None) -> float: """Return how many seconds *request* must still be held individually - because of its :reqmeta:`throttling_delay`, or ``0.0`` if it has none + because of its :reqmeta:`delay`, or ``0.0`` if it has none or it has already elapsed. The one-time delay is started on the first call. Unlike a scope delay, this affects only *request*: a - :ref:`throttling-aware scheduler ` must + :ref:`throttler-aware scheduler ` must hold the request back on its own, **without** blocking other requests that share its scopes. """ @@ -423,7 +423,7 @@ class ThrottlingManagerProtocol(Protocol): consumed: float | None = None, remaining: float | None = None, ) -> None: - """Reconcile the :ref:`throttling quota ` of each of + """Reconcile the :ref:`throttler quota ` of each of *scopes* with an actually *consumed* amount (a delta to add) or a *remaining* amount (an absolute value), correcting the estimate used when requests were sent. @@ -444,8 +444,8 @@ class ThrottlingManagerProtocol(Protocol): directly (e.g. an adaptive-delay extension). """ - def get_scope_manager(self, scope_id: str) -> ThrottlingScopeManagerProtocol: - """Return the :class:`ThrottlingScopeManagerProtocol` instance handling + def get_scope_manager(self, scope_id: str) -> ThrottlerScopeManagerProtocol: + """Return the :class:`ThrottlerScopeManagerProtocol` instance handling the scope identified by *scope_id*, creating it if necessary.""" @@ -456,16 +456,16 @@ _GetScopesMethod = TypeVar( # Request.meta key under which scope_cache persists the resolved scopes so that # they survive a request being serialized to and restored from a disk queue. -_RESOLVED_SCOPES_META_KEY = "_throttling_resolved_scopes" +_RESOLVED_SCOPES_META_KEY = "_throttler_resolved_scopes" def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: - """Decorator for :meth:`~ThrottlingManagerProtocol.get_scopes` + """Decorator for :meth:`~ThrottlerProtocol.get_scopes` implementations that persists the resolved scopes on ``request.meta``. The readers of the resolved scopes — the synchronous readiness API of a - :ref:`throttling-aware scheduler ` and - :meth:`~ThrottlingManagerProtocol.get_resolved_scopes` — read this persisted + :ref:`throttler-aware scheduler ` and + :meth:`~ThrottlerProtocol.get_resolved_scopes` — read this persisted value instead of resolving the scopes again, so they stay cheap and consistent, and it survives a request being serialized to and restored from a :ref:`disk queue ` (which is @@ -483,10 +483,10 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: .. code-block:: python from scrapy.utils.httpobj import urlparse_cached - from scrapy.throttling import scope_cache + from scrapy.throttler import scope_cache - class MyThrottlingManager: + class MyThrottler: @scope_cache async def get_scopes(self, request): return urlparse_cached(request).hostname or "" @@ -505,8 +505,8 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: return wrapper # type: ignore[return-value] -class ThrottlingManager: - """The default :setting:`THROTTLING_MANAGER` class. +class Throttler: + """The default :setting:`THROTTLER` class. It assigns to each request its domain or subdomain as scope and handles backoff according to :ref:`backoff settings `. @@ -520,13 +520,13 @@ class ThrottlingManager: self.crawler = crawler _warn_on_deprecated_concurrency(crawler.settings) _warn_on_unachievable_concurrency(crawler.settings) - self._debug = crawler.settings.getbool("THROTTLING_DEBUG") - self._max_idle = crawler.settings.getfloat("THROTTLING_SCOPE_MAX_IDLE") + self._debug = crawler.settings.getbool("THROTTLER_DEBUG") + self._max_idle = crawler.settings.getfloat("THROTTLER_SCOPE_MAX_IDLE") self._robotstxt_obey = crawler.settings.getbool( "ROBOTSTXT_OBEY" - ) and crawler.settings.getbool("THROTTLING_ROBOTSTXT_OBEY") + ) and crawler.settings.getbool("THROTTLER_ROBOTSTXT_OBEY") self._robotstxt_max_delay = crawler.settings.getfloat( - "THROTTLING_ROBOTSTXT_MAX_DELAY" + "THROTTLER_ROBOTSTXT_MAX_DELAY" ) self._default_useragent: str = crawler.settings["USER_AGENT"] self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] @@ -535,40 +535,40 @@ class ThrottlingManager: self._on_robots_parsed, signal=signals.robots_parsed ) self._default_scope_manager_cls = load_object( - crawler.settings["THROTTLING_SCOPE_MANAGER"] + crawler.settings["THROTTLER_SCOPE_MANAGER"] ) self._scopes_config: dict[str, dict[str, Any]] = self._merge_download_slots( crawler.settings ) # Ordered by least-recently-used first (see get_scope_manager), so the - # scope limit can evict the coldest idle scopes (see THROTTLING_SCOPE_LIMIT). - self._scope_managers: OrderedDict[ScopeID, ThrottlingScopeManagerProtocol] = ( + # scope limit can evict the coldest idle scopes (see THROTTLER_SCOPE_LIMIT). + self._scope_managers: OrderedDict[ScopeID, ThrottlerScopeManagerProtocol] = ( OrderedDict() ) - self._scope_limit: int = crawler.settings.getint("THROTTLING_SCOPE_LIMIT") + self._scope_limit: int = crawler.settings.getint("THROTTLER_SCOPE_LIMIT") self._last_eviction: float | None = None # Concurrency slots reserved by acquire(), to be released once the # request finishes downloading. self._reserved: WeakKeyDictionary[ - Request, list[tuple[ThrottlingScopeManagerProtocol, float | None]] + Request, list[tuple[ThrottlerScopeManagerProtocol, float | None]] ] = WeakKeyDictionary() @staticmethod def _merge_download_slots(settings: BaseSettings) -> dict[str, dict[str, Any]]: """Return the effective per-scope configuration, merging the deprecated - :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLING_SCOPES`. + :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLER_SCOPES`. - Each ``DOWNLOAD_SLOTS`` entry is translated to a throttling scope keyed + Each ``DOWNLOAD_SLOTS`` entry is translated to a throttler scope keyed by the same slot name (the default manager keys domain scopes by host name, which is what download slots used too): ``concurrency`` and ``delay`` map directly, and the ``randomize_delay`` boolean maps to a ``jitter`` magnitude (the historical ±50%, or none). An explicit - ``THROTTLING_SCOPES`` entry for the same scope takes precedence over the + ``THROTTLER_SCOPES`` entry for the same scope takes precedence over the translated one. The deprecation warning is emitted by the downloader. """ scopes: dict[str, dict[str, Any]] = { scope_id: dict(config) - for scope_id, config in settings.getdict("THROTTLING_SCOPES").items() + for scope_id, config in settings.getdict("THROTTLER_SCOPES").items() } for slot_id, slot_config in settings.getdict("DOWNLOAD_SLOTS").items(): translated: dict[str, Any] = {} @@ -578,7 +578,7 @@ class ThrottlingManager: translated["delay"] = slot_config["delay"] if "randomize_delay" in slot_config: translated["jitter"] = 0.5 if slot_config["randomize_delay"] else 0.0 - # An explicit THROTTLING_SCOPES entry wins over the translated one. + # An explicit THROTTLER_SCOPES entry wins over the translated one. scopes[slot_id] = {**translated, **scopes.get(slot_id, {})} return scopes @@ -597,14 +597,14 @@ class ThrottlingManager: :func:`scope_cache`). Subclasses whose :meth:`get_scopes` cannot be resolved synchronously rely on that persisted value instead. """ - scopes = request.meta.get("throttling_scopes") + scopes = request.meta.get("throttler_scopes") if scopes is not None: return cast("RequestScopes", scopes) download_slot = request.meta.get("download_slot") if download_slot is not None: warnings.warn( "The 'download_slot' request meta key is deprecated. Use " - "'throttling_scopes' instead.", + "'throttler_scopes' instead.", category=ScrapyDeprecationWarning, stacklevel=2, ) @@ -641,7 +641,7 @@ class ThrottlingManager: # -- Scope-state coordination (called from the request lifecycle) -------- - def get_scope_manager(self, scope_id: ScopeID) -> ThrottlingScopeManagerProtocol: + def get_scope_manager(self, scope_id: ScopeID) -> ThrottlerScopeManagerProtocol: manager = self._scope_managers.get(scope_id) if manager is not None: # Mark as most-recently-used for the LRU scope limit. @@ -655,7 +655,7 @@ class ThrottlingManager: else self._default_scope_manager_cls ) manager = cast( - "ThrottlingScopeManagerProtocol", + "ThrottlerScopeManagerProtocol", build_from_crawler(manager_cls, self.crawler, config), ) self._scope_managers[scope_id] = manager @@ -664,7 +664,7 @@ class ThrottlingManager: def _enforce_scope_limit(self, keep: ScopeID) -> None: """Evict least-recently-used idle scopes while the number of live scope - managers exceeds :setting:`THROTTLING_SCOPE_LIMIT` (``0`` disables the + managers exceeds :setting:`THROTTLER_SCOPE_LIMIT` (``0`` disables the limit). LRU order is kept by :meth:`get_scope_manager` moving each accessed @@ -684,7 +684,7 @@ class ThrottlingManager: del self._scope_managers[scope_id] async def acquire(self, request: Request) -> None: - # A throttling-aware scheduler reserves the request before handing it + # A throttler-aware scheduler reserves the request before handing it # to the engine, so there is nothing left to wait for or record here. if request in self._reserved: return @@ -728,7 +728,7 @@ class ThrottlingManager: def _record_reservation( self, request: Request, - managers: list[tuple[ThrottlingScopeManagerProtocol, float | None]], + managers: list[tuple[ThrottlerScopeManagerProtocol, float | None]], ) -> None: """Record a send on each of *request*'s scope *managers* and mark *request* as reserved, so :meth:`release` can later free the slots. This @@ -744,7 +744,7 @@ class ThrottlingManager: for manager, _ in managers: manager.record_done() - # -- Synchronous readiness API (used by a throttling-aware scheduler) ------ + # -- Synchronous readiness API (used by a throttler-aware scheduler) ------ def is_ready(self, request: Request) -> bool: now = time.monotonic() @@ -759,7 +759,7 @@ class ThrottlingManager: return True def reserve(self, request: Request) -> None: - # A throttling-aware scheduler reserves every request before handing it + # A throttler-aware scheduler reserves every request before handing it # to the engine, so acquire() always returns early for it and never # gets to evict idle scopes; do it here so their managers do not pile # up on broad crawls. @@ -789,8 +789,8 @@ class ThrottlingManager: """Block until any of *managers* frees a concurrency slot. Each manager hands out an event Deferred that fires when a slot is freed - (via :meth:`ThrottlingScopeManager.record_done`) or the limit is raised - (via :meth:`ThrottlingScopeManager.set_concurrency`). A long safety timer + (via :meth:`ThrottlerScopeManager.record_done`) or the limit is raised + (via :meth:`ThrottlerScopeManager.set_concurrency`). A long safety timer bounds the wait in case no slot is ever freed (it always should be, via :meth:`release`). """ @@ -802,7 +802,7 @@ class ThrottlingManager: manager.discard_slot_event(event) async def _delay_request(self, request: Request) -> None: - """Honor the :reqmeta:`throttling_delay` meta key by holding *request* + """Honor the :reqmeta:`delay` meta key by holding *request* for the requested number of seconds the first time it is processed. This is the blocking (:meth:`acquire`) counterpart of @@ -815,31 +815,31 @@ class ThrottlingManager: if wait <= 0: return await sleep(wait) - request.meta["_throttling_delayed"] = True + request.meta["_throttler_delayed"] = True def _request_delay_deadline(self, request: Request, now: float) -> float: """Return the monotonic time before which *request* must not be sent due - to its :reqmeta:`throttling_delay`, or ``0.0`` if it has none. + to its :reqmeta:`delay`, or ``0.0`` if it has none. This is the readiness-API counterpart of :meth:`_delay_request`: - a throttling-aware scheduler gates requests through :meth:`is_ready` and + a throttler-aware scheduler gates requests through :meth:`is_ready` and :meth:`get_time_until_ready` instead of awaiting :meth:`acquire`, so the delay is enforced by holding back the request until this deadline rather than by sleeping. The deadline is computed once, the first time the request reaches the gate, and stored so later polls reuse it. - A request whose delay has already been honored (the ``_throttling_delayed`` + A request whose delay has already been honored (the ``_throttler_delayed`` flag, also set by :meth:`_delay_request`) is never delayed again, which keeps a resumed crawl from re-blocking on a stale deadline.""" - delay = request.meta.get("throttling_delay") - if not delay or request.meta.get("_throttling_delayed"): + delay = request.meta.get("delay") + if not delay or request.meta.get("_throttler_delayed"): return 0.0 - deadline = request.meta.get("_throttling_delay_deadline") + deadline = request.meta.get("_throttler_delay_deadline") if deadline is None: deadline = now + float(delay) - request.meta["_throttling_delay_deadline"] = deadline + request.meta["_throttler_delay_deadline"] = deadline if self._debug: - logger.debug(f"Holding {request} for {delay:.2f}s (throttling_delay)") + logger.debug(f"Holding {request} for {delay:.2f}s (delay)") return deadline def back_off( @@ -889,11 +889,11 @@ class ThrottlingManager: def apply_robots_crawl_delay(self, scope_id: ScopeID, delay: float) -> None: """Honor a robots.txt ``Crawl-delay`` directive of *delay* seconds for *scope_id* by setting its delay (capped at - :setting:`THROTTLING_ROBOTSTXT_MAX_DELAY`) and its concurrency to ``1``. + :setting:`THROTTLER_ROBOTSTXT_MAX_DELAY`) and its concurrency to ``1``. Called from the :signal:`robots_parsed` signal handler when - :setting:`THROTTLING_ROBOTSTXT_OBEY` is enabled. An explicit - :setting:`THROTTLING_SCOPES` configuration for the scope is respected, but + :setting:`THROTTLER_ROBOTSTXT_OBEY` is enabled. An explicit + :setting:`THROTTLER_SCOPES` configuration for the scope is respected, but a warning is logged about the discrepancy unless its ``ignore_robots_txt`` key is ``True``. """ @@ -910,10 +910,10 @@ class ThrottlingManager: conflicts.append(f"concurrency={config['concurrency']!r} > 1") if conflicts: logger.warning( - f"Throttling scope {scope_id!r} is configured with {' and '.join(conflicts)}, " + f"Throttler scope {scope_id!r} is configured with {' and '.join(conflicts)}, " f"which is more aggressive than its robots.txt Crawl-delay of " f"{capped}s. The configured values take precedence; set " - "'ignore_robots_txt': True in its THROTTLING_SCOPES entry to " + "'ignore_robots_txt': True in its THROTTLER_SCOPES entry to " "silence this warning." ) return @@ -951,12 +951,12 @@ class ThrottlingManager: del self._scope_managers[scope_id] -class ThrottlingScopeManagerProtocol(Protocol): - """A protocol for :setting:`THROTTLING_SCOPE_MANAGER` :ref:`components +class ThrottlerScopeManagerProtocol(Protocol): + """A protocol for :setting:`THROTTLER_SCOPE_MANAGER` :ref:`components `. The ``__init__`` method gets a ``config`` dict with the base configuration - of the managed throttling scope. For example: + of the managed throttler scope. For example: .. code-block:: python @@ -988,7 +988,7 @@ class ThrottlingScopeManagerProtocol(Protocol): """Return the number of seconds to wait before a request for this scope may be sent, or ``0`` if it may be sent right away. - *amount* is the expected :ref:`throttling quota ` + *amount* is the expected :ref:`throttler quota ` consumption of the request, if any. """ @@ -996,7 +996,7 @@ class ThrottlingScopeManagerProtocol(Protocol): self, now: float | None = None, amount: float | None = None ) -> None: """Record that a request for this scope has just been sent, consuming - *amount* of its :ref:`throttling quota ` if given.""" + *amount* of its :ref:`throttler quota ` if given.""" def record_done(self, now: float | None = None) -> None: """Record that a previously :meth:`record_sent` request has finished @@ -1017,7 +1017,7 @@ class ThrottlingScopeManagerProtocol(Protocol): *cap* limits *delay* to :setting:`BACKOFF_MAX_DELAY`. It is ``True`` for untrusted input such as response headers, and may be set to ``False`` for trusted, programmatic delays (see - :meth:`ThrottlingManagerProtocol.delay_scope`). + :meth:`ThrottlerProtocol.delay_scope`). """ def reconcile_quota( @@ -1026,7 +1026,7 @@ class ThrottlingScopeManagerProtocol(Protocol): remaining: float | None = None, now: float | None = None, ) -> None: - """Reconcile the :ref:`throttling quota ` of this + """Reconcile the :ref:`throttler quota ` of this scope with the actual *consumed* amount (or the *remaining* amount) reported for a request, correcting the estimate used by :meth:`record_sent`.""" @@ -1049,7 +1049,7 @@ class ThrottlingScopeManagerProtocol(Protocol): def concurrency_blocked(self) -> bool: """Return whether this scope is at its concurrency limit. - :class:`ThrottlingManager` calls this (after all time-based gates in + :class:`Throttler` calls this (after all time-based gates in :meth:`can_send` are open) to decide whether to wait for a freed slot. Return ``False`` when no concurrency limit is enforced. """ @@ -1058,7 +1058,7 @@ class ThrottlingScopeManagerProtocol(Protocol): """Return the current load of this scope: a non-negative number, with ``1.0`` meaning "as busy as its concurrency limit allows". - A :ref:`throttling-aware scheduler ` uses + A :ref:`throttler-aware scheduler ` uses this to break ties between equally-prioritized requests, preferring the least-loaded scopes. The reference implementation returns active sends divided by the concurrency limit (falling back to @@ -1076,7 +1076,7 @@ class ThrottlingScopeManagerProtocol(Protocol): def discard_slot_event(self, event: Deferred[None]) -> None: """Cancel a pending slot event returned by :meth:`slot_event`. - Called by :class:`ThrottlingManager` when the wait ends without the + Called by :class:`Throttler` when the wait ends without the event firing (e.g. another scope's slot opened first). """ @@ -1095,8 +1095,8 @@ class ThrottlingScopeManagerProtocol(Protocol): _SLOT_WAIT_TIMEOUT = 1.0 -class ThrottlingScopeManager: - """The default :setting:`THROTTLING_SCOPE_MANAGER` class. +class ThrottlerScopeManager: + """The default :setting:`THROTTLER_SCOPE_MANAGER` class. It implements a per-scope state machine covering delay, exponential :ref:`backoff `, concurrency and :ref:`quotas @@ -1122,7 +1122,7 @@ class ThrottlingScopeManager: than that many requests are allowed in flight at once. - When the scope is configured with a ``"quota"``, no more than that much - quota is consumed per ``"window"`` (default: :setting:`THROTTLING_WINDOW`). + quota is consumed per ``"window"`` (default: :setting:`THROTTLER_WINDOW`). """ @classmethod @@ -1134,7 +1134,7 @@ class ThrottlingScopeManager: backoff: dict[str, Any] = config.get("backoff", {}) self._id: ScopeID = config.get("id", "") # The per-scope delay defaults to DOWNLOAD_DELAY; a scope can override - # it with its own "delay" config (see THROTTLING_SCOPES). + # it with its own "delay" config (see THROTTLER_SCOPES). self._base_delay: float = float( config.get("delay", settings.getfloat("DOWNLOAD_DELAY")) ) @@ -1180,7 +1180,7 @@ class ThrottlingScopeManager: quota = config.get("quota") self._quota: float | None = None if quota is None else float(quota) self._quota_window: float = float( - config.get("window", settings.getfloat("THROTTLING_WINDOW")) + config.get("window", settings.getfloat("THROTTLER_WINDOW")) ) # State. diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b69733dc..4ac67da50 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -75,9 +75,9 @@ def get_crawler( if prevent_warnings: # Pin the per-scope concurrency to CONCURRENT_REQUESTS_PER_DOMAIN's # default: keeps the suite's historical behavior and silences the - # warn-then-flip warning (see throttling._warn_on_deprecated_concurrency). + # warn-then-flip warning (see throttler._warn_on_deprecated_concurrency). settings.setdefault( - "THROTTLING_SCOPE_CONCURRENCY", + "THROTTLER_SCOPE_CONCURRENCY", default_settings.CONCURRENT_REQUESTS_PER_DOMAIN, ) runner: CrawlerRunnerBase diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 578b2396a..9b1264e76 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -55,12 +55,12 @@ def get_raw_crawler( class TestScopeConcurrencyBridge: """CONCURRENT_REQUESTS_PER_DOMAIN is deprecated and bridged into - THROTTLING_SCOPE_CONCURRENCY by setting priority (see - scrapy.throttling._default_scope_concurrency).""" + THROTTLER_SCOPE_CONCURRENCY by setting priority (see + scrapy.throttler._default_scope_concurrency).""" @staticmethod def _resolve(settings: Settings) -> int: - from scrapy.throttling import _default_scope_concurrency # noqa: PLC0415 + from scrapy.throttler import _default_scope_concurrency # noqa: PLC0415 return _default_scope_concurrency(settings) @@ -75,18 +75,18 @@ class TestScopeConcurrencyBridge: def test_scope_concurrency_overrides(self) -> None: settings = Settings() - settings.set("THROTTLING_SCOPE_CONCURRENCY", 4, priority="project") + settings.set("THROTTLER_SCOPE_CONCURRENCY", 4, priority="project") assert self._resolve(settings) == 4 def test_new_setting_wins_on_non_default_tie(self) -> None: settings = Settings() settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="project") - settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project") + settings.set("THROTTLER_SCOPE_CONCURRENCY", 9, priority="project") assert self._resolve(settings) == 9 def test_higher_priority_deprecated_setting_wins(self) -> None: settings = Settings() - settings.set("THROTTLING_SCOPE_CONCURRENCY", 9, priority="project") + settings.set("THROTTLER_SCOPE_CONCURRENCY", 9, priority="project") settings.set("CONCURRENT_REQUESTS_PER_DOMAIN", 4, priority="cmdline") assert self._resolve(settings) == 4 @@ -111,7 +111,7 @@ class TestScopeConcurrencyBridge: with warnings.catch_warnings(record=True) as recorded: warnings.simplefilter("always") get_crawler( - settings_dict={"THROTTLING_SCOPE_CONCURRENCY": 4}, + settings_dict={"THROTTLER_SCOPE_CONCURRENCY": 4}, prevent_warnings=False, ) messages = [str(w.message) for w in recorded] diff --git a/tests/test_engine.py b/tests/test_engine.py index 4bca525c5..e4f44ca0a 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -655,7 +655,7 @@ async def test_request_scheduled_signal(): crawler.signals.disconnect(signal_handler, signals.request_scheduled) -class TestEngineThrottling: +class TestEngineThrottler: @pytest.fixture def engine(self): crawler = get_crawler(MySpider) @@ -708,25 +708,25 @@ class TestEngineThrottling: engine._maybe_arm_delay_wakeup() assert engine._delay_wakeup is None - def test_maybe_warn_throttling_backout(self, engine): - # A scheduler without get_next_request_delay is not throttling-aware, so + def test_maybe_warn_throttler_backout(self, engine): + # A scheduler without get_next_request_delay is not throttler-aware, so # the warning recommends switching to one. engine._get_next_request_delay = None with LogCapture() as log: - engine._maybe_warn_throttling_backout() + engine._maybe_warn_throttler_backout() # A second call is a no-op (the warning is emitted only once). - engine._maybe_warn_throttling_backout() - assert engine._throttling_backout_warned is True - assert str(log).count("ThrottlingAwareScheduler") == 1 + engine._maybe_warn_throttler_backout() + assert engine._throttler_backout_warned is True + assert str(log).count("ThrottlerAwareScheduler") == 1 - def test_maybe_warn_throttling_backout_throttling_aware(self, engine): - # A throttling-aware scheduler (one with get_next_request_delay) holds + def test_maybe_warn_throttler_backout_throttler_aware(self, engine): + # A throttler-aware scheduler (one with get_next_request_delay) holds # throttled requests itself, so no warning is emitted. engine._get_next_request_delay = lambda: None with LogCapture() as log: - engine._maybe_warn_throttling_backout() - assert engine._throttling_backout_warned is False - assert "ThrottlingAwareScheduler" not in str(log) + engine._maybe_warn_throttler_backout() + assert engine._throttler_backout_warned is False + assert "ThrottlerAwareScheduler" not in str(log) def test_spider_is_idle_false_while_scheduling(self, engine): engine._slot = Mock() @@ -734,7 +734,7 @@ class TestEngineThrottling: engine.scraper.slot.is_idle.return_value = True engine.downloader = Mock() engine.downloader.active = [] - engine._throttling_waiting = set() + engine._throttler_waiting = set() engine._start = None engine._scheduling = 1 # An in-flight async enqueue keeps the spider from being considered idle. diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py index cf7f94f04..cff66bb1e 100644 --- a/tests/test_pqueues.py +++ b/tests/test_pqueues.py @@ -8,11 +8,11 @@ from scrapy.http.request import Request from scrapy.pqueues import ( DownloaderAwarePriorityQueue, ScrapyPriorityQueue, - ThrottlingAwarePriorityQueue, + ThrottlerAwarePriorityQueue, ) from scrapy.spiders import Spider from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue -from scrapy.throttling import iter_scopes +from scrapy.throttler import iter_scopes from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.test import get_crawler from tests.test_scheduler import MockDownloader @@ -191,22 +191,22 @@ class TestDownloaderAwarePriorityQueue: # No active downloads are tracked in the downloader, so every slot has # the same score and tie-breaking must not starve a slot. req_a1 = Request("https://example.org/a1") - req_a1.meta["throttling_scopes"] = "slot-a" + req_a1.meta["throttler_scopes"] = "slot-a" req_b1 = Request("https://example.org/b1") - req_b1.meta["throttling_scopes"] = "slot-b" + req_b1.meta["throttler_scopes"] = "slot-b" req_a2 = Request("https://example.org/a2") - req_a2.meta["throttling_scopes"] = "slot-a" + req_a2.meta["throttler_scopes"] = "slot-a" req_b2 = Request("https://example.org/b2") - req_b2.meta["throttling_scopes"] = "slot-b" + req_b2.meta["throttler_scopes"] = "slot-b" for request in (req_a1, req_b1, req_a2, req_b2): self.queue.push(request) slots = [ - self.queue.pop().meta["throttling_scopes"], - self.queue.pop().meta["throttling_scopes"], - self.queue.pop().meta["throttling_scopes"], - self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], ] assert slots == ["slot-a", "slot-b", "slot-a", "slot-b"] @@ -215,22 +215,22 @@ class TestDownloaderAwarePriorityQueue: # If the selected slot becomes empty, rotation should continue from # that slot marker to avoid restarting from the smallest slot. req_a1 = Request("https://example.org/a1") - req_a1.meta["throttling_scopes"] = "slot-a" + req_a1.meta["throttler_scopes"] = "slot-a" req_a2 = Request("https://example.org/a2") - req_a2.meta["throttling_scopes"] = "slot-a" + req_a2.meta["throttler_scopes"] = "slot-a" req_b1 = Request("https://example.org/b1") - req_b1.meta["throttling_scopes"] = "slot-b" + req_b1.meta["throttler_scopes"] = "slot-b" req_c1 = Request("https://example.org/c1") - req_c1.meta["throttling_scopes"] = "slot-c" + req_c1.meta["throttler_scopes"] = "slot-c" for request in (req_a1, req_a2, req_b1, req_c1): self.queue.push(request) slots = [ - self.queue.pop().meta["throttling_scopes"], - self.queue.pop().meta["throttling_scopes"], - self.queue.pop().meta["throttling_scopes"], - self.queue.pop().meta["throttling_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], + self.queue.pop().meta["throttler_scopes"], ] assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"] @@ -240,11 +240,11 @@ class TestDownloaderAwarePriorityQueue: assert throttler is not None req_a = Request("https://example.org/a") - req_a.meta["throttling_scopes"] = "slot-a" + req_a.meta["throttler_scopes"] = "slot-a" req_b = Request("https://example.org/b") - req_b.meta["throttling_scopes"] = "slot-b" + req_b.meta["throttler_scopes"] = "slot-b" req_c = Request("https://example.org/c") - req_c.meta["throttling_scopes"] = "slot-c" + req_c.meta["throttler_scopes"] = "slot-c" for req in (req_a, req_b, req_c): self.queue.push(req) @@ -257,7 +257,7 @@ class TestDownloaderAwarePriorityQueue: def test_contains(self): req = Request("https://example.org/") - req.meta["throttling_scopes"] = "example-slot" + req.meta["throttler_scopes"] = "example-slot" assert "example-slot" not in self.queue self.queue.push(req) assert "example-slot" in self.queue @@ -317,10 +317,10 @@ def test_pop_order(input_, output): assert actual_output_urls == expected_output_urls -class TestThrottlingAwarePriorityQueue: +class TestThrottlerAwarePriorityQueue: def _queue(self, crawler, key=""): return build_from_crawler( - ThrottlingAwarePriorityQueue, + ThrottlerAwarePriorityQueue, crawler, downstream_queue_cls=FifoMemoryQueue, key=key, @@ -346,7 +346,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler( Spider, settings_dict={ - "THROTTLING_SCOPES": {"slow.com": {"delay": 1000.0}}, + "THROTTLER_SCOPES": {"slow.com": {"delay": 1000.0}}, "RANDOMIZE_DOWNLOAD_DELAY": False, }, ) @@ -369,13 +369,13 @@ class TestThrottlingAwarePriorityQueue: assert delay == pytest.approx(1000.0, abs=1.0) @coroutine_test - async def test_pop_holds_request_with_throttling_delay(self): + async def test_pop_holds_request_with_delay(self): crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) queue = self._queue(crawler) await self._push( queue, crawler, - Request("http://slow.com/1", meta={"throttling_delay": 1000.0}), + Request("http://slow.com/1", meta={"delay": 1000.0}), ) await self._push(queue, crawler, Request("http://fast.com/1")) # The delayed request is held back even though its scope is otherwise @@ -398,7 +398,7 @@ class TestThrottlingAwarePriorityQueue: await self._push( queue, crawler, - Request("http://example.com/slow", meta={"throttling_delay": 1000.0}), + Request("http://example.com/slow", meta={"delay": 1000.0}), ) await self._push(queue, crawler, Request("http://example.com/fast")) # The delayed request is held aside, so the other request in the same @@ -413,7 +413,7 @@ class TestThrottlingAwarePriorityQueue: async def test_delayed_request_promoted_when_due(self): crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) queue = self._queue(crawler) - request = Request("http://example.com/slow", meta={"throttling_delay": 1000.0}) + request = Request("http://example.com/slow", meta={"delay": 1000.0}) await self._push(queue, crawler, request) assert queue.pop() is None # held back by its per-request delay # Once the delay elapses the request is promoted into its scope-set @@ -421,7 +421,7 @@ class TestThrottlingAwarePriorityQueue: queue._promote_ready(queue._delayed[0][0]) popped = queue.pop() assert popped is request - assert popped.meta["_throttling_delayed"] is True + assert popped.meta["_throttler_delayed"] is True assert len(queue) == 0 @coroutine_test @@ -432,7 +432,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler(Spider, settings_dict={"RANDOMIZE_DOWNLOAD_DELAY": False}) temp_dir = tempfile.mkdtemp() queue = build_from_crawler( - ThrottlingAwarePriorityQueue, + ThrottlerAwarePriorityQueue, crawler, downstream_queue_cls=PickleFifoDiskQueue, key=temp_dir, @@ -440,13 +440,13 @@ class TestThrottlingAwarePriorityQueue: await self._push( queue, crawler, - Request("http://example.com/slow", meta={"throttling_delay": 1000.0}), + Request("http://example.com/slow", meta={"delay": 1000.0}), ) assert len(queue) == 1 # held in memory, not yet in any scope-set queue state = queue.close() # graceful stop resumed = build_from_crawler( - ThrottlingAwarePriorityQueue, + ThrottlerAwarePriorityQueue, crawler, downstream_queue_cls=PickleFifoDiskQueue, key=temp_dir, @@ -457,7 +457,7 @@ class TestThrottlingAwarePriorityQueue: assert popped is not None assert popped.url == "http://example.com/slow" # Its delay is marked consumed, so it does not re-block on resume. - assert popped.meta["_throttling_delayed"] is True + assert popped.meta["_throttler_delayed"] is True resumed.close() @coroutine_test @@ -465,7 +465,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler( Spider, settings_dict={ - "THROTTLING_SCOPES": { + "THROTTLER_SCOPES": { "a.com": {"concurrency": 4}, "b.com": {"concurrency": 4}, } @@ -486,7 +486,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler( Spider, settings_dict={ - "THROTTLING_SCOPES": { + "THROTTLER_SCOPES": { "a.com": {"concurrency": 4}, "b.com": {"concurrency": 4}, } @@ -512,18 +512,6 @@ class TestThrottlingAwarePriorityQueue: await self._push(queue, crawler, Request("http://a.com/1")) assert queue.close() != {} - @coroutine_test - async def test_peek(self): - crawler = get_crawler(Spider) - queue = self._queue(crawler) - assert queue.peek() is None - await self._push(queue, crawler, Request("http://a.com/1")) - head = queue.peek() - assert head is not None - assert head.url == "http://a.com/1" - # peek() does not consume the request. - assert len(queue) == 1 - @coroutine_test async def test_get_next_request_delay_zero_when_ready(self): crawler = get_crawler(Spider) @@ -537,7 +525,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler(Spider) queue = self._queue(crawler) # An empty (but still registered) internal queue is skipped. - queue.pqueues[frozenset({"a.com"})] = queue.pqfactory(frozenset({"a.com"})) + queue.pqueues[frozenset({"a.com"})] = queue._pqfactory(frozenset({"a.com"})) assert queue.get_next_request_delay() is None @coroutine_test @@ -545,7 +533,7 @@ class TestThrottlingAwarePriorityQueue: crawler = get_crawler( Spider, settings_dict={ - "THROTTLING_SCOPES": { + "THROTTLER_SCOPES": { "a.com": {"delay": 10.0}, "b.com": {"delay": 1000.0}, }, @@ -572,27 +560,16 @@ class TestThrottlingAwarePriorityQueue: queue = self._queue(crawler) await self._push(queue, crawler, Request("http://a.com/1")) inner = next(iter(queue.pqueues.values())) - # peek() still reports a sendable head, but pop() yields nothing: the + # _select() still reports a sendable head, but pop() yields nothing: the # request-is-None guard must not try to reserve a missing request. inner.pop = lambda: None assert queue.pop() is None - @coroutine_test - async def test_contains(self): - crawler = get_crawler(Spider) - queue = self._queue(crawler) - scope_set = frozenset( - iter_scopes(await crawler.throttler.get_scopes(Request("http://a.com/1"))) - ) - assert scope_set not in queue - await self._push(queue, crawler, Request("http://a.com/1")) - assert scope_set in queue - def test_non_dict_slot_startprios(self): crawler = get_crawler(Spider) with pytest.raises(ValueError, match="slot_startprios"): build_from_crawler( - ThrottlingAwarePriorityQueue, + ThrottlerAwarePriorityQueue, crawler, downstream_queue_cls=FifoMemoryQueue, key="", diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index b8b1f8c7d..05dda6026 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -11,7 +11,7 @@ from unittest.mock import Mock import pytest from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import BaseScheduler, Scheduler, ThrottlingAwareScheduler +from scrapy.core.scheduler import BaseScheduler, Scheduler, ThrottlerAwareScheduler from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -411,22 +411,22 @@ class TestIncompatibility: self._incompatible() -_THROTTLING_AWARE_PQ = "scrapy.pqueues.ThrottlingAwarePriorityQueue" +_THROTTLER_AWARE_PQ = "scrapy.pqueues.ThrottlerAwarePriorityQueue" -class TestThrottlingAwareScheduler: +class TestThrottlerAwareScheduler: def _crawler(self, settings_dict: dict[str, Any] | None = None) -> Crawler: settings = { - "SCHEDULER_PRIORITY_QUEUE": _THROTTLING_AWARE_PQ, + "SCHEDULER_PRIORITY_QUEUE": _THROTTLER_AWARE_PQ, "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", **(settings_dict or {}), } return get_crawler(Spider, settings) - def _scheduler(self, crawler: Crawler) -> ThrottlingAwareScheduler: + def _scheduler(self, crawler: Crawler) -> ThrottlerAwareScheduler: spider = Spider(name="spider") crawler.spider = spider - scheduler = ThrottlingAwareScheduler.from_crawler(crawler) + scheduler = ThrottlerAwareScheduler.from_crawler(crawler) scheduler.open(spider) return scheduler @@ -449,21 +449,21 @@ class TestThrottlingAwareScheduler: scheduler.enqueue_request(Request("http://a.com/1")) scheduler.close("finished") - def test_requires_throttling_aware_priority_queue(self) -> None: + def test_requires_throttler_aware_priority_queue(self) -> None: crawler = self._crawler( {"SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ScrapyPriorityQueue"} ) spider = Spider(name="spider") crawler.spider = spider - scheduler = ThrottlingAwareScheduler.from_crawler(crawler) - with pytest.raises(ValueError, match="throttling-aware priority queue"): + scheduler = ThrottlerAwareScheduler.from_crawler(crawler) + with pytest.raises(ValueError, match="throttler-aware priority queue"): scheduler.open(spider) @coroutine_test async def test_delay_blocks_and_reports_delay(self) -> None: crawler = self._crawler( { - "THROTTLING_SCOPES": {"slow.com": {"delay": 1000.0}}, + "THROTTLER_SCOPES": {"slow.com": {"delay": 1000.0}}, "RANDOMIZE_DOWNLOAD_DELAY": False, }, ) @@ -481,7 +481,7 @@ class TestThrottlingAwareScheduler: @coroutine_test async def test_no_delay_when_only_concurrency_blocked(self) -> None: crawler = self._crawler( - {"THROTTLING_SCOPES": {"slow.com": {"concurrency": 1}}}, + {"THROTTLER_SCOPES": {"slow.com": {"concurrency": 1}}}, ) scheduler = self._scheduler(crawler) await scheduler.enqueue_request_async(Request("http://slow.com/1")) @@ -494,14 +494,14 @@ class TestThrottlingAwareScheduler: @coroutine_test async def test_delayed_request_survives_jobdir_stop(self, tmp_path: Path) -> None: - # A request held back by its per-request throttling_delay must not be + # A request held back by its per-request delay must not be # lost on a graceful stop when a JOBDIR is configured: it is flushed to # the disk queue on close and restored on resume. crawler = self._crawler( {"JOBDIR": str(tmp_path), "RANDOMIZE_DOWNLOAD_DELAY": False} ) scheduler = self._scheduler(crawler) - request = Request("http://a.com/slow", meta={"throttling_delay": 1000.0}) + request = Request("http://a.com/slow", meta={"delay": 1000.0}) assert await scheduler.enqueue_request_async(request) is True assert len(scheduler) == 1 # The delay holds it back, so nothing is dequeued before the stop. @@ -585,14 +585,14 @@ class TestThrottlingAwareScheduler: scheduler2.close("finished") -class TestIntegrationWithThrottlingAwareScheduler: +class TestIntegrationWithThrottlerAwareScheduler: @inline_callbacks_test def test_integration(self): crawler = get_crawler( spidercls=StartUrlsSpider, settings_dict={ - "SCHEDULER": "scrapy.core.scheduler.ThrottlingAwareScheduler", - "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlingAwarePriorityQueue", + "SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue", "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", }, ) @@ -628,8 +628,8 @@ class TestIntegrationWithThrottlingAwareScheduler: crawler = get_crawler( spidercls=FollowSpider, settings_dict={ - "SCHEDULER": "scrapy.core.scheduler.ThrottlingAwareScheduler", - "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlingAwarePriorityQueue", + "SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue", "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", }, ) @@ -638,16 +638,16 @@ class TestIntegrationWithThrottlingAwareScheduler: @inline_callbacks_test def test_integration_with_delay(self): - # A small per-scope delay forces the engine to arm the throttling wakeup + # A small per-scope delay forces the engine to arm the throttler wakeup # timer between requests; the crawl must still complete. crawler = get_crawler( spidercls=StartUrlsSpider, settings_dict={ - "SCHEDULER": "scrapy.core.scheduler.ThrottlingAwareScheduler", - "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlingAwarePriorityQueue", + "SCHEDULER": "scrapy.core.scheduler.ThrottlerAwareScheduler", + "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.ThrottlerAwarePriorityQueue", "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "RANDOMIZE_DOWNLOAD_DELAY": False, - "THROTTLING_SCOPES": {"127.0.0.1": {"delay": 0.05}}, + "THROTTLER_SCOPES": {"127.0.0.1": {"delay": 0.05}}, }, ) with MockServer() as mockserver: diff --git a/tests/test_throttling.py b/tests/test_throttler.py similarity index 90% rename from tests/test_throttling.py rename to tests/test_throttler.py index b05181a29..503ecc26d 100644 --- a/tests/test_throttling.py +++ b/tests/test_throttler.py @@ -8,9 +8,9 @@ from scrapy import signals from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response from scrapy.settings import default_settings -from scrapy.throttling import ( - ThrottlingManager, - ThrottlingScopeManager, +from scrapy.throttler import ( + Throttler, + ThrottlerScopeManager, _add_bare_scope, _parse_ratelimit_reset, _parse_retry_after, @@ -30,12 +30,12 @@ from tests.utils.decorators import coroutine_test def _manager(settings=None): crawler = get_crawler(settings_dict=settings) - return ThrottlingManager.from_crawler(crawler) + return Throttler.from_crawler(crawler) def _scope_manager(settings=None, config=None): crawler = get_crawler(settings_dict=settings) - return ThrottlingScopeManager.from_crawler(crawler, config or {"id": "example.com"}) + return ThrottlerScopeManager.from_crawler(crawler, config or {"id": "example.com"}) class _FakeRobotParser: @@ -67,11 +67,11 @@ def test_deprecated_concurrency_defaults_differ(): shipping a bogus warning or aborting a crawl.""" assert ( default_settings.CONCURRENT_REQUESTS_PER_DOMAIN - != default_settings.THROTTLING_SCOPE_CONCURRENCY + != default_settings.THROTTLER_SCOPE_CONCURRENCY ) -class TestThrottlingManager: +class TestThrottler: @coroutine_test async def test_get_scopes_returns_netloc(self): manager = _manager() @@ -91,20 +91,20 @@ class TestThrottlingManager: @coroutine_test async def test_get_scopes_meta_string(self): manager = _manager() - request = Request("http://example.com/a", meta={"throttling_scopes": "api"}) + request = Request("http://example.com/a", meta={"throttler_scopes": "api"}) assert await manager.get_scopes(request) == "api" @coroutine_test async def test_get_scopes_meta_dict(self): manager = _manager() request = Request( - "http://example.com/a", meta={"throttling_scopes": {"api": 2.0}} + "http://example.com/a", meta={"throttler_scopes": {"api": 2.0}} ) assert await manager.get_scopes(request) == {"api": 2.0} @coroutine_test async def test_get_scopes_persisted_in_meta(self): - from scrapy.throttling import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 + from scrapy.throttler import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 manager = _manager() request = Request("http://example.com/a") @@ -114,7 +114,7 @@ class TestThrottlingManager: @coroutine_test async def test_scope_cache_works_without_crawler(self): # scope_cache only persists to meta; it needs nothing from the manager. - from scrapy.throttling import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 + from scrapy.throttler import _RESOLVED_SCOPES_META_KEY # noqa: PLC0415 class CrawlerlessManager: @scope_cache @@ -131,7 +131,7 @@ class TestThrottlingManager: # path reuses them instead of resolving again. calls = [] - class CountingManager(ThrottlingManager): + class CountingManager(Throttler): @scope_cache async def get_scopes(self, request): calls.append(request.url) @@ -155,7 +155,7 @@ class TestThrottlingManager: manager = _manager() request = Request( - "http://example.com/a", meta={"throttling_scopes": {"bucket": 3.0}} + "http://example.com/a", meta={"throttler_scopes": {"bucket": 3.0}} ) await manager.get_scopes(request) # A request restored from a disk queue is a fresh object; the synchronous @@ -183,13 +183,13 @@ class TestThrottlingManager: def test_scope_manager_class_in_config(self): manager = _manager( - {"THROTTLING_SCOPES": {"example.com": {"manager": ThrottlingScopeManager}}} + {"THROTTLER_SCOPES": {"example.com": {"manager": ThrottlerScopeManager}}} ) scope = manager._get_scope_manager("example.com") - assert isinstance(scope, ThrottlingScopeManager) + assert isinstance(scope, ThrottlerScopeManager) def test_release_frees_concurrency(self): - manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}}) scope = manager._get_scope_manager("example.com") request = Request("http://example.com") scope.record_sent(now=0.0) @@ -205,7 +205,7 @@ class TestThrottlingManager: async def test_acquire_waits_for_freed_slot(self): from scrapy.utils.asyncio import call_later # noqa: PLC0415 - manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}}) r1 = Request("http://example.com/1") r2 = Request("http://example.com/2") # Drive acquire() the way the engine does, so it runs as a real task that @@ -222,7 +222,7 @@ class TestThrottlingManager: assert r2 in manager._reserved def test_apply_backoff_reconciles_quota_without_backoff(self): - manager = _manager({"THROTTLING_SCOPES": {"cost": {"quota": 100.0}}}) + manager = _manager({"THROTTLER_SCOPES": {"cost": {"quota": 100.0}}}) scope = manager._get_scope_manager("cost") manager._apply_backoff({"cost": {"consumed": 5.0}}) # Quota was reconciled but no backoff step was applied. @@ -230,7 +230,7 @@ class TestThrottlingManager: assert scope._backoff_level == 0 def test_apply_backoff_delay_and_consumed(self): - manager = _manager({"THROTTLING_SCOPES": {"cost": {"quota": 100.0}}}) + manager = _manager({"THROTTLER_SCOPES": {"cost": {"quota": 100.0}}}) scope = manager._get_scope_manager("cost") manager._apply_backoff({"cost": {"delay": 5.0, "consumed": 2.0}}) assert scope._consumed == pytest.approx(2.0) @@ -317,7 +317,7 @@ class TestThrottlingManager: ("settings", "parser_delay", "expected_base_delay"), [ ({"ROBOTSTXT_OBEY": True, "RANDOMIZE_DOWNLOAD_DELAY": False}, 3.0, 3.0), - ({"ROBOTSTXT_OBEY": True, "THROTTLING_ROBOTSTXT_OBEY": False}, 3.0, 0.0), + ({"ROBOTSTXT_OBEY": True, "THROTTLER_ROBOTSTXT_OBEY": False}, 3.0, 0.0), ({"ROBOTSTXT_OBEY": True}, None, 0.0), ({"ROBOTSTXT_OBEY": True}, ValueError(), 0.0), ], @@ -344,13 +344,13 @@ class TestThrottlingManager: def test_apply_robots_crawl_delay_capped(self): manager = _manager( - {"ROBOTSTXT_OBEY": True, "THROTTLING_ROBOTSTXT_MAX_DELAY": 2.0} + {"ROBOTSTXT_OBEY": True, "THROTTLER_ROBOTSTXT_MAX_DELAY": 2.0} ) manager.apply_robots_crawl_delay("example.com", 30.0) assert manager._get_scope_manager("example.com")._base_delay == 2.0 def test_apply_robots_crawl_delay_disabled(self): - manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLING_ROBOTSTXT_OBEY": False}) + manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLER_ROBOTSTXT_OBEY": False}) manager.apply_robots_crawl_delay("example.com", 3.0) assert manager._get_scope_manager("example.com")._base_delay == 0.0 @@ -363,10 +363,10 @@ class TestThrottlingManager: manager = _manager( { "ROBOTSTXT_OBEY": True, - "THROTTLING_SCOPES": {"example.com": {"concurrency": 8}}, + "THROTTLER_SCOPES": {"example.com": {"concurrency": 8}}, } ) - with caplog.at_level(logging.WARNING, logger="scrapy.throttling"): + with caplog.at_level(logging.WARNING, logger="scrapy.throttler"): manager.apply_robots_crawl_delay("example.com", 3.0) assert "Crawl-delay" in caplog.text # The configured value takes precedence (crawl-delay not applied). @@ -376,19 +376,19 @@ class TestThrottlingManager: manager = _manager( { "ROBOTSTXT_OBEY": True, - "THROTTLING_SCOPES": { + "THROTTLER_SCOPES": { "example.com": {"concurrency": 8, "ignore_robots_txt": True} }, } ) - with caplog.at_level(logging.WARNING, logger="scrapy.throttling"): + with caplog.at_level(logging.WARNING, logger="scrapy.throttler"): manager.apply_robots_crawl_delay("example.com", 3.0) # No warning is logged and the crawl-delay is not applied. assert "Crawl-delay" not in caplog.text assert manager._get_scope_manager("example.com")._base_delay == 0.0 def test_scope_eviction(self): - manager = _manager({"THROTTLING_SCOPE_MAX_IDLE": 100.0}) + manager = _manager({"THROTTLER_SCOPE_MAX_IDLE": 100.0}) scope = manager._get_scope_manager("example.com") scope.record_sent(now=0.0) scope.record_done(now=0.0) @@ -403,22 +403,22 @@ class TestThrottlingManager: def test_scope_eviction_skips_active_backoff(self): manager = _manager( - {"THROTTLING_SCOPE_MAX_IDLE": 100.0, "BACKOFF_MAX_DELAY": 100_000.0} + {"THROTTLER_SCOPE_MAX_IDLE": 100.0, "BACKOFF_MAX_DELAY": 100_000.0} ) scope = manager._get_scope_manager("example.com") scope.record_backoff(delay=10_000.0, now=0.0) manager._last_eviction = None manager._maybe_evict(now=5_000.0) # Still in backoff (in_backoff_until far in the future), so not evicted - # even though it has been idle for longer than THROTTLING_SCOPE_MAX_IDLE. + # even though it has been idle for longer than THROTTLER_SCOPE_MAX_IDLE. assert "example.com" in manager._scope_managers def test_reserve_evicts_idle_scopes(self): - # A throttling-aware scheduler reserves every request before the engine + # A throttler-aware scheduler reserves every request before the engine # reaches acquire() (which fast-paths reserved requests), so reserve() # must be the hook that evicts idle scope managers; otherwise they pile # up unbounded on broad crawls. - manager = _manager({"THROTTLING_SCOPE_MAX_IDLE": 1.0}) + manager = _manager({"THROTTLER_SCOPE_MAX_IDLE": 1.0}) idle = manager._get_scope_manager("idle.example") # Make it look long-idle: a finished send in the distant monotonic past. idle.record_sent(now=0.0) @@ -429,7 +429,7 @@ class TestThrottlingManager: assert "active.example" in manager._scope_managers def test_scope_limit_evicts_least_recently_used(self): - manager = _manager({"THROTTLING_SCOPE_LIMIT": 2}) + manager = _manager({"THROTTLER_SCOPE_LIMIT": 2}) # Use three scopes in order; each send/done leaves them idle. for scope_id in ("a.example", "b.example", "c.example"): scope = manager._get_scope_manager(scope_id) @@ -439,7 +439,7 @@ class TestThrottlingManager: assert set(manager._scope_managers) == {"b.example", "c.example"} def test_scope_limit_keeps_active_scopes(self): - manager = _manager({"THROTTLING_SCOPE_LIMIT": 1}) + manager = _manager({"THROTTLER_SCOPE_LIMIT": 1}) # Two scopes with in-flight requests cannot be evicted, so the limit is # exceeded rather than dropping a scope that still tracks a live send. for scope_id in ("a.example", "b.example"): @@ -447,7 +447,7 @@ class TestThrottlingManager: assert set(manager._scope_managers) == {"a.example", "b.example"} def test_scope_limit_disabled(self): - manager = _manager({"THROTTLING_SCOPE_LIMIT": 0}) + manager = _manager({"THROTTLER_SCOPE_LIMIT": 0}) for i in range(5): scope = manager._get_scope_manager(f"{i}.example") scope.record_sent(now=0.0) @@ -455,7 +455,7 @@ class TestThrottlingManager: assert len(manager._scope_managers) == 5 -class TestThrottlingScopeManager: +class TestThrottlerScopeManager: def test_no_delay_by_default(self): scope = _scope_manager() scope.record_sent(now=0.0) @@ -588,10 +588,10 @@ class TestThrottlingScopeManager: assert scope._concurrency == 8 def test_no_scope_concurrency_limit_when_zero(self): - # THROTTLING_SCOPE_CONCURRENCY governs scopes that are neither a domain + # THROTTLER_SCOPE_CONCURRENCY governs scopes that are neither a domain # nor an IP (here a bare "custom" group name). scope = _scope_manager( - settings={"THROTTLING_SCOPE_CONCURRENCY": 0}, config={"id": "custom"} + settings={"THROTTLER_SCOPE_CONCURRENCY": 0}, config={"id": "custom"} ) assert scope._concurrency is None for _ in range(100): @@ -702,8 +702,8 @@ class TestThrottlingScopeManager: assert scope._consumed == pytest.approx(7.0) -class TestThrottlingManagerReadiness: - """The synchronous readiness API used by a throttling-aware scheduler.""" +class TestThrottlerReadiness: + """The synchronous readiness API used by a throttler-aware scheduler.""" @coroutine_test async def test_is_ready_unconstrained_scope(self): @@ -725,7 +725,7 @@ class TestThrottlingManagerReadiness: async def test_reserve_blocks_delay_scope(self): manager = _manager( { - "THROTTLING_SCOPES": {"example.com": {"delay": 100.0}}, + "THROTTLER_SCOPES": {"example.com": {"delay": 100.0}}, "RANDOMIZE_DOWNLOAD_DELAY": False, } ) @@ -740,28 +740,28 @@ class TestThrottlingManagerReadiness: assert manager.get_time_until_ready(second) == pytest.approx(100.0, abs=1.0) @coroutine_test - async def test_throttling_delay_blocks_until_deadline(self): - manager = _manager({"THROTTLING_DEBUG": True}) - request = Request("http://example.com/a", meta={"throttling_delay": 100.0}) + async def test_delay_blocks_until_deadline(self): + manager = _manager({"THROTTLER_DEBUG": True}) + request = Request("http://example.com/a", meta={"delay": 100.0}) await manager.get_scopes(request) # The per-request delay holds back the request even though its scope is # otherwise unconstrained. assert manager.is_ready(request) is False assert manager.get_time_until_ready(request) == pytest.approx(100.0, abs=1.0) # The deadline is computed once and reused by later polls. - deadline = request.meta["_throttling_delay_deadline"] + deadline = request.meta["_throttler_delay_deadline"] assert manager.is_ready(request) is False - assert request.meta["_throttling_delay_deadline"] == deadline + assert request.meta["_throttler_delay_deadline"] == deadline @coroutine_test - async def test_throttling_delay_not_reapplied_once_consumed(self): + async def test_delay_not_reapplied_once_consumed(self): # A request whose delay was already honored (e.g. promoted out of a - # throttling-aware queue's holding area, or restored on resume) is + # throttler-aware queue's holding area, or restored on resume) is # ready, so it cannot re-block its scope set on a stale deadline. manager = _manager() request = Request( "http://example.com/a", - meta={"throttling_delay": 100.0, "_throttling_delayed": True}, + meta={"delay": 100.0, "_throttler_delayed": True}, ) await manager.get_scopes(request) assert manager.is_ready(request) is True @@ -771,16 +771,14 @@ class TestThrottlingManagerReadiness: async def test_get_request_delay(self): manager = _manager() assert manager.get_request_delay( - Request("http://example.com/a", meta={"throttling_delay": 100.0}) + Request("http://example.com/a", meta={"delay": 100.0}) ) == pytest.approx(100.0, abs=1.0) # A request without a per-request delay is not held individually. assert manager.get_request_delay(Request("http://example.com/b")) == 0.0 @coroutine_test async def test_delay_scope(self): - manager = _manager( - {"THROTTLING_DEBUG": True, "RANDOMIZE_DOWNLOAD_DELAY": False} - ) + manager = _manager({"THROTTLER_DEBUG": True, "RANDOMIZE_DOWNLOAD_DELAY": False}) request = Request("http://example.com/a") await manager.get_scopes(request) assert manager.is_ready(request) is True @@ -804,7 +802,7 @@ class TestThrottlingManagerReadiness: @coroutine_test async def test_reserve_blocks_on_concurrency(self): - manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}}) + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}}) first = Request("http://example.com/1") second = Request("http://example.com/2") await manager.get_scopes(first) @@ -820,7 +818,7 @@ class TestThrottlingManagerReadiness: async def test_acquire_noop_when_reserved(self): manager = _manager( { - "THROTTLING_SCOPES": {"example.com": {"delay": 100.0}}, + "THROTTLER_SCOPES": {"example.com": {"delay": 100.0}}, "RANDOMIZE_DOWNLOAD_DELAY": False, } ) @@ -835,7 +833,7 @@ class TestThrottlingManagerReadiness: @coroutine_test async def test_get_scope_load(self): - manager = _manager({"THROTTLING_SCOPES": {"example.com": {"concurrency": 4}}}) + manager = _manager({"THROTTLER_SCOPES": {"example.com": {"concurrency": 4}}}) assert manager.get_scope_load("example.com") == 0.0 request = Request("http://example.com/1") await manager.get_scopes(request) @@ -937,11 +935,11 @@ class TestScopeHelpers: update_scope_backoff({"a": 1.0}, "a", delay=1.0) -class TestThrottlingManagerEdges: +class TestThrottlerEdges: @coroutine_test async def test_acquire_without_scopes(self): manager = _manager() - request = Request("http://example.com/a", meta={"throttling_scopes": []}) + request = Request("http://example.com/a", meta={"throttler_scopes": []}) # No scopes resolve, so acquire() returns without reserving anything. await manager.acquire(request) assert request not in manager._reserved @@ -956,9 +954,9 @@ class TestThrottlingManagerEdges: async def test_acquire_logs_and_waits_for_delay(self): manager = _manager( { - "THROTTLING_SCOPES": {"example.com": {"delay": 0.02}}, + "THROTTLER_SCOPES": {"example.com": {"delay": 0.02}}, "RANDOMIZE_DOWNLOAD_DELAY": False, - "THROTTLING_DEBUG": True, + "THROTTLER_DEBUG": True, } ) r1 = Request("http://example.com/1") @@ -974,8 +972,8 @@ class TestThrottlingManagerEdges: manager = _manager( { - "THROTTLING_SCOPES": {"example.com": {"concurrency": 1}}, - "THROTTLING_DEBUG": True, + "THROTTLER_SCOPES": {"example.com": {"concurrency": 1}}, + "THROTTLER_DEBUG": True, } ) r1 = Request("http://example.com/1") @@ -988,10 +986,10 @@ class TestThrottlingManagerEdges: @coroutine_test async def test_delay_request(self): - manager = _manager({"THROTTLING_DEBUG": True}) - request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) + manager = _manager({"THROTTLER_DEBUG": True}) + request = Request("http://example.com/a", meta={"delay": 0.01}) await manager._delay_request(request) - assert request.meta["_throttling_delayed"] is True + assert request.meta["_throttler_delayed"] is True # A second call is a no-op (the request was already delayed). await manager._delay_request(request) @@ -1000,9 +998,9 @@ class TestThrottlingManagerEdges: # Same as above but with debug logging off, so the delay is applied # without emitting the debug message. manager = _manager() - request = Request("http://example.com/a", meta={"throttling_delay": 0.01}) + request = Request("http://example.com/a", meta={"delay": 0.01}) await manager._delay_request(request) - assert request.meta["_throttling_delayed"] is True + assert request.meta["_throttler_delayed"] is True @coroutine_test async def test_wait_for_slot_discards_unfired_events(self): @@ -1028,14 +1026,14 @@ class TestThrottlingManagerEdges: assert manager._get_scope_manager("example.com")._backoff_level == 1 def test_apply_backoff_debug_logging(self, caplog): - manager = _manager({"THROTTLING_DEBUG": True}) - with caplog.at_level(logging.DEBUG, logger="scrapy.throttling"): + manager = _manager({"THROTTLER_DEBUG": True}) + with caplog.at_level(logging.DEBUG, logger="scrapy.throttler"): manager._apply_backoff("example.com") assert "Backoff for scope" in caplog.text assert manager._get_scope_manager("example.com")._backoff_level == 1 def test_on_robots_parsed_disabled(self): - manager = _manager({"THROTTLING_ROBOTSTXT_OBEY": False}) + manager = _manager({"THROTTLER_ROBOTSTXT_OBEY": False}) # With obeying disabled, the handler returns without touching any scope. manager._on_robots_parsed(_FakeRobotParser(5.0), Request("http://example.com")) assert not manager._scope_managers @@ -1044,21 +1042,21 @@ class TestThrottlingManagerEdges: manager = _manager( { "ROBOTSTXT_OBEY": True, - "THROTTLING_SCOPES": {"example.com": {"delay": 0.5}}, + "THROTTLER_SCOPES": {"example.com": {"delay": 0.5}}, } ) - with caplog.at_level(logging.WARNING, logger="scrapy.throttling"): + with caplog.at_level(logging.WARNING, logger="scrapy.throttler"): manager.apply_robots_crawl_delay("example.com", 3.0) assert "Crawl-delay" in caplog.text def test_apply_robots_crawl_delay_debug_logging(self, caplog): - manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLING_DEBUG": True}) - with caplog.at_level(logging.DEBUG, logger="scrapy.throttling"): + manager = _manager({"ROBOTSTXT_OBEY": True, "THROTTLER_DEBUG": True}) + with caplog.at_level(logging.DEBUG, logger="scrapy.throttler"): manager.apply_robots_crawl_delay("example.com", 3.0) assert "Crawl-delay" in caplog.text def test_maybe_evict_disabled(self): - manager = _manager({"THROTTLING_SCOPE_MAX_IDLE": 0}) + manager = _manager({"THROTTLER_SCOPE_MAX_IDLE": 0}) scope = manager._get_scope_manager("example.com") scope.record_sent(now=0.0) scope.record_done(now=0.0) @@ -1067,7 +1065,7 @@ class TestThrottlingManagerEdges: assert "example.com" in manager._scope_managers -class TestThrottlingScopeManagerEdges: +class TestThrottlerScopeManagerEdges: def test_jitter_as_range(self): scope = _scope_manager( config={"id": "x", "backoff": {"jitter": [0.0, 0.0]}, "min_delay": 1.0} @@ -1134,7 +1132,7 @@ class TestThrottlingScopeManagerEdges: assert scope.is_idle(now=0.0, max_idle=1.0) is True -class TestThrottlingIntegration: +class TestThrottlerIntegration: @coroutine_test async def test_backoff_recorded_on_429(self, mockserver): crawler = get_crawler(SimpleSpider, {"RETRY_ENABLED": False}) @@ -1144,21 +1142,21 @@ class TestThrottlingIntegration: throttler = crawler.throttler assert throttler is not None managers = throttler._scope_managers - assert managers, "no throttling scope was created" + assert managers, "no throttler scope was created" assert any(manager._backoff_level >= 1 for manager in managers.values()) @coroutine_test async def test_backoff_recorded_on_download_error(self, mockserver): crawler = get_crawler(SimpleSpider, {"RETRY_ENABLED": False}) # A dropped connection raises a DownloadFailedError, which the engine - # routes through the throttling manager before re-raising. + # routes through the throttler before re-raising. await crawler.crawl_async( mockserver.url("/drop?abort=1"), mockserver=mockserver ) throttler = crawler.throttler assert throttler is not None managers = throttler._scope_managers - assert managers, "no throttling scope was created" + assert managers, "no throttler scope was created" assert any(manager._backoff_level >= 1 for manager in managers.values()) @coroutine_test