mirror of https://github.com/scrapy/scrapy.git
WIP
This commit is contained in:
Adrián Chaves
parent
07fd9265d0
commit
3224b3669c
12
docs/conf.py
12
docs/conf.py
|
|
@ -34,6 +34,7 @@ extensions = [
|
|||
"sphinx.ext.coverage",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx_reredirects",
|
||||
"sphinx_rtd_dark_mode",
|
||||
]
|
||||
|
||||
|
|
@ -136,6 +137,12 @@ coverage_ignore_pyobjects = [
|
|||
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
|
||||
]
|
||||
|
||||
# -- Options for the autodoc extension ----------------------------------------
|
||||
autodoc_type_aliases = {
|
||||
"BackoffData": "BackoffData",
|
||||
"GetScopesMethod": "GetScopesMethod",
|
||||
"RequestScopes": "RequestScopes",
|
||||
}
|
||||
|
||||
# -- Options for the InterSphinx extension -----------------------------------
|
||||
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
|
||||
|
|
@ -157,6 +164,11 @@ intersphinx_mapping = {
|
|||
}
|
||||
intersphinx_disabled_reftypes: Sequence[str] = []
|
||||
|
||||
# -- sphinx-reredirects -------------------------------------------------------
|
||||
redirects = {
|
||||
"topics/autothrottle": "throttling.html",
|
||||
}
|
||||
|
||||
|
||||
# -- Options for sphinx-hoverxref extension ----------------------------------
|
||||
# https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html
|
||||
|
|
|
|||
|
|
@ -161,12 +161,11 @@ Solving specific problems
|
|||
topics/leaks
|
||||
topics/media-pipeline
|
||||
topics/deploy
|
||||
topics/autothrottle
|
||||
topics/throttling
|
||||
topics/benchmarking
|
||||
topics/jobs
|
||||
topics/coroutines
|
||||
topics/asyncio
|
||||
topics/throttling
|
||||
|
||||
:doc:`faq`
|
||||
Get answers to most frequently asked questions.
|
||||
|
|
@ -198,8 +197,9 @@ Solving specific problems
|
|||
:doc:`topics/deploy`
|
||||
Deploying your Scrapy spiders and run them in a remote server.
|
||||
|
||||
:doc:`topics/autothrottle`
|
||||
Adjust crawl rate dynamically based on load.
|
||||
:doc:`topics/throttling`
|
||||
Control request throttling to avoid overloading websites and comply with
|
||||
rate limits.
|
||||
|
||||
:doc:`topics/benchmarking`
|
||||
Check how Scrapy performs on your hardware.
|
||||
|
|
@ -213,10 +213,6 @@ Solving specific problems
|
|||
:doc:`topics/asyncio`
|
||||
Use :mod:`asyncio` and :mod:`asyncio`-powered libraries.
|
||||
|
||||
:doc:`topics/throttling`
|
||||
Control request throttling to avoid overloading websites and comply with
|
||||
rate limits.
|
||||
|
||||
.. _extending-scrapy:
|
||||
|
||||
Extending Scrapy
|
||||
|
|
|
|||
|
|
@ -81,11 +81,9 @@ error happens while handling it.
|
|||
|
||||
While this enables you to do very fast crawls (sending multiple concurrent
|
||||
requests at the same time, in a fault-tolerant way) Scrapy also gives you
|
||||
control over the politeness of the crawl through :ref:`a few settings
|
||||
<topics-settings-ref>`. You can do things like setting a download delay between
|
||||
each request, limiting the amount of concurrent requests per domain or per IP, and
|
||||
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
||||
to figure these settings out automatically.
|
||||
control over :ref:`throttling <throttling>`, e.g. you can set a delay between
|
||||
requests to the same domain, set a maximum concurrency per domain, or customize
|
||||
the :ref:`backoff <backoff>` behavior.
|
||||
|
||||
.. note::
|
||||
|
||||
|
|
|
|||
|
|
@ -969,10 +969,9 @@ New features
|
|||
:meth:`~scrapy.crawler.Crawler.get_spider_middleware`.
|
||||
(:issue:`6181`)
|
||||
|
||||
- Slot delay updates by the :ref:`AutoThrottle extension
|
||||
<topics-autothrottle>` based on response latencies can now be disabled for
|
||||
specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta
|
||||
key.
|
||||
- Slot delay updates by the ``scrapy.extensions.throttle.AutoThrottle``
|
||||
extension based on response latencies can now be disabled for specific
|
||||
requests via the ``autothrottle_dont_adjust_delay`` meta key.
|
||||
(:issue:`6246`, :issue:`6527`)
|
||||
|
||||
- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``,
|
||||
|
|
@ -4943,8 +4942,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
|
|||
|
||||
* From :class:`~scrapy.spiders.Spider` (and subclasses):
|
||||
|
||||
* ``DOWNLOAD_DELAY`` (use :ref:`download_delay
|
||||
<spider-download_delay-attribute>`)
|
||||
* ``DOWNLOAD_DELAY`` (use ``download_delay``)
|
||||
|
||||
* ``set_crawler`` (use :meth:`~scrapy.spiders.Spider.from_crawler`)
|
||||
|
||||
|
|
@ -7121,7 +7119,7 @@ Scrapy changes:
|
|||
|
||||
- added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way
|
||||
- added options ``-o`` and ``-t`` to the :command:`runspider` command
|
||||
- documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
|
||||
- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
|
||||
- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
|
||||
- added a ``process_start_requests()`` method to spider middlewares
|
||||
- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
sphinx==8.1.3
|
||||
sphinx-hoverxref==1.4.2
|
||||
sphinx-notfound-page==1.0.4
|
||||
sphinx-reredirects==1.0.0
|
||||
sphinx-rtd-theme==3.0.2
|
||||
sphinx-rtd-dark-mode==1.3.0
|
||||
|
|
|
|||
|
|
@ -1,195 +0,0 @@
|
|||
.. _topics-autothrottle:
|
||||
|
||||
======================
|
||||
AutoThrottle extension
|
||||
======================
|
||||
|
||||
This is an extension for automatically throttling crawling speed based on load
|
||||
of both the Scrapy server and the website you are crawling.
|
||||
|
||||
Design goals
|
||||
============
|
||||
|
||||
1. be nicer to sites instead of using default download delay of zero
|
||||
2. automatically adjust Scrapy to the optimum crawling speed, so the user
|
||||
doesn't have to tune the download delays to find the optimum one.
|
||||
The user only needs to specify the maximum concurrent requests
|
||||
it allows, and the extension does the rest.
|
||||
|
||||
.. _autothrottle-algorithm:
|
||||
|
||||
How it works
|
||||
============
|
||||
|
||||
Scrapy allows defining the concurrency and delay of different download slots,
|
||||
e.g. through the :setting:`DOWNLOAD_SLOTS` setting. By default requests are
|
||||
assigned to slots based on their URL domain, although it is possible to
|
||||
customize the download slot of any request.
|
||||
|
||||
The AutoThrottle extension adjusts the delay of each download slot dynamically,
|
||||
to make your spider send :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` concurrent
|
||||
requests on average to each remote website.
|
||||
|
||||
It uses download latency to compute the delays. The main idea is the
|
||||
following: if a server needs ``latency`` seconds to respond, a client
|
||||
should send a request each ``latency/N`` seconds to have ``N`` requests
|
||||
processed in parallel.
|
||||
|
||||
Instead of adjusting the delays one can just set a small fixed
|
||||
download delay and impose hard limits on concurrency using
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar
|
||||
effect, but there are some important differences:
|
||||
|
||||
* because the download delay is small there will be occasional bursts
|
||||
of requests;
|
||||
* often non-200 (error) responses can be returned faster than regular
|
||||
responses, so with a small download delay and a hard concurrency limit
|
||||
crawler will be sending requests to server faster when server starts to
|
||||
return errors. But this is an opposite of what crawler should do - in case
|
||||
of errors it makes more sense to slow down: these errors may be caused by
|
||||
the high request rate.
|
||||
|
||||
AutoThrottle doesn't have these issues.
|
||||
|
||||
Throttling algorithm
|
||||
====================
|
||||
|
||||
AutoThrottle algorithm adjusts download delays based on the following rules:
|
||||
|
||||
1. spiders always start with a download delay of
|
||||
:setting:`AUTOTHROTTLE_START_DELAY`;
|
||||
2. when a response is received, the target download delay is calculated as
|
||||
``latency / N`` where ``latency`` is a latency of the response,
|
||||
and ``N`` is :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`.
|
||||
3. download delay for next requests is set to the average of previous
|
||||
download delay and the target download delay;
|
||||
4. latencies of non-200 responses are not allowed to decrease the delay;
|
||||
5. download delay can't become less than :setting:`DOWNLOAD_DELAY` or greater
|
||||
than :setting:`AUTOTHROTTLE_MAX_DELAY`
|
||||
|
||||
.. note:: The AutoThrottle extension honours the standard Scrapy settings for
|
||||
concurrency and delay. This means that it will respect
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP` options and
|
||||
never set a download delay lower than :setting:`DOWNLOAD_DELAY`.
|
||||
|
||||
.. _download-latency:
|
||||
|
||||
In Scrapy, the download latency is measured as the time elapsed between
|
||||
establishing the TCP connection and receiving the HTTP headers.
|
||||
|
||||
Note that these latencies are very hard to measure accurately in a cooperative
|
||||
multitasking environment because Scrapy may be busy processing a spider
|
||||
callback, for example, and unable to attend downloads. However, these latencies
|
||||
should still give a reasonable estimate of how busy Scrapy (and ultimately, the
|
||||
server) is, and this extension builds on that premise.
|
||||
|
||||
.. reqmeta:: autothrottle_dont_adjust_delay
|
||||
|
||||
Prevent specific requests from triggering slot delay adjustments
|
||||
================================================================
|
||||
|
||||
AutoThrottle adjusts the delay of download slots based on the latencies of
|
||||
responses that belong to that download slot. The only exceptions are non-200
|
||||
responses, which are only taken into account to increase that delay, but
|
||||
ignored if they would decrease that delay.
|
||||
|
||||
You can also set the ``autothrottle_dont_adjust_delay`` request metadata key to
|
||||
``True`` in any request to prevent its response latency from impacting the
|
||||
delay of its download slot:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy import Request
|
||||
|
||||
Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True})
|
||||
|
||||
Note, however, that AutoThrottle still determines the starting delay of every
|
||||
download slot by setting the ``download_delay`` attribute on the running
|
||||
spider. If you want AutoThrottle not to impact a download slot at all, in
|
||||
addition to setting this meta key in all requests that use that download slot,
|
||||
you might want to set a custom value for the ``delay`` attribute of that
|
||||
download slot, e.g. using :setting:`DOWNLOAD_SLOTS`.
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
The settings used to control the AutoThrottle extension are:
|
||||
|
||||
* :setting:`AUTOTHROTTLE_ENABLED`
|
||||
* :setting:`AUTOTHROTTLE_START_DELAY`
|
||||
* :setting:`AUTOTHROTTLE_MAX_DELAY`
|
||||
* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
|
||||
* :setting:`AUTOTHROTTLE_DEBUG`
|
||||
* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
* :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||
* :setting:`DOWNLOAD_DELAY`
|
||||
|
||||
For more information see :ref:`autothrottle-algorithm`.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_ENABLED
|
||||
|
||||
AUTOTHROTTLE_ENABLED
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Enables the AutoThrottle extension.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_START_DELAY
|
||||
|
||||
AUTOTHROTTLE_START_DELAY
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``5.0``
|
||||
|
||||
The initial download delay (in seconds).
|
||||
|
||||
.. setting:: AUTOTHROTTLE_MAX_DELAY
|
||||
|
||||
AUTOTHROTTLE_MAX_DELAY
|
||||
~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``60.0``
|
||||
|
||||
The maximum download delay (in seconds) to be set in case of high latencies.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_TARGET_CONCURRENCY
|
||||
|
||||
AUTOTHROTTLE_TARGET_CONCURRENCY
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``1.0``
|
||||
|
||||
Average number of requests Scrapy should be sending in parallel to remote
|
||||
websites. It must be higher than ``0.0``.
|
||||
|
||||
By default, AutoThrottle adjusts the delay to send a single
|
||||
concurrent request to each of the remote websites. Set this option to
|
||||
a higher value (e.g. ``2.0``) to increase the throughput and the load on remote
|
||||
servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value
|
||||
(e.g. ``0.5``) makes the crawler more conservative and polite.
|
||||
|
||||
Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected
|
||||
when AutoThrottle extension is enabled. This means that if
|
||||
``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number
|
||||
of concurrent requests.
|
||||
|
||||
At every given time point Scrapy can be sending more or less concurrent
|
||||
requests than ``AUTOTHROTTLE_TARGET_CONCURRENCY``; it is a suggested
|
||||
value the crawler tries to approach, not a hard limit.
|
||||
|
||||
.. setting:: AUTOTHROTTLE_DEBUG
|
||||
|
||||
AUTOTHROTTLE_DEBUG
|
||||
~~~~~~~~~~~~~~~~~~
|
||||
|
||||
Default: ``False``
|
||||
|
||||
Enable AutoThrottle debug mode which will display stats on every response
|
||||
received, so you can see how the throttling parameters are being adjusted in
|
||||
real time.
|
||||
|
|
@ -629,7 +629,6 @@ are some special keys recognized by Scrapy and its built-in extensions.
|
|||
Those are:
|
||||
|
||||
* :reqmeta:`allow_offsite`
|
||||
* :reqmeta:`autothrottle_dont_adjust_delay`
|
||||
* :reqmeta:`bindaddress`
|
||||
* :reqmeta:`cookiejar`
|
||||
* :reqmeta:`dont_cache`
|
||||
|
|
|
|||
|
|
@ -502,6 +502,17 @@ Default: ``100``
|
|||
Maximum number of concurrent items (per response) to process in parallel in
|
||||
:ref:`item pipelines <topics-item-pipeline>`.
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS
|
||||
|
||||
CONCURRENT_REQUESTS
|
||||
-------------------
|
||||
|
||||
Default: ``16``
|
||||
|
||||
Maximum number of total concurrent requests allowed.
|
||||
|
||||
.. seealso:: :ref:`throttling`
|
||||
|
||||
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
|
||||
|
||||
DEFAULT_DROPITEM_LOG_LEVEL
|
||||
|
|
@ -1128,7 +1139,6 @@ Default:
|
|||
"scrapy.extensions.feedexport.FeedExporter": 0,
|
||||
"scrapy.extensions.logstats.LogStats": 0,
|
||||
"scrapy.extensions.spiderstate.SpiderState": 0,
|
||||
"scrapy.extensions.throttle.AutoThrottle": 0,
|
||||
}
|
||||
|
||||
A dict containing the extensions available by default in Scrapy, and their
|
||||
|
|
@ -1509,26 +1519,6 @@ Example::
|
|||
|
||||
NEWSPIDER_MODULE = 'mybot.spiders_dev'
|
||||
|
||||
.. setting:: RANDOMIZE_DOWNLOAD_DELAY
|
||||
|
||||
RANDOMIZE_DOWNLOAD_DELAY
|
||||
------------------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
If enabled, Scrapy will wait a random amount of time (between 0.5 * :setting:`DOWNLOAD_DELAY` and 1.5 * :setting:`DOWNLOAD_DELAY`) while fetching requests from the same
|
||||
website.
|
||||
|
||||
This randomization decreases the chance of the crawler being detected (and
|
||||
subsequently blocked) by sites which analyze requests looking for statistically
|
||||
significant similarities in the time between their requests.
|
||||
|
||||
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
||||
|
||||
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||
|
||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||
|
||||
.. setting:: REACTOR_THREADPOOL_MAXSIZE
|
||||
|
||||
REACTOR_THREADPOOL_MAXSIZE
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -47,6 +47,7 @@ if TYPE_CHECKING:
|
|||
|
||||
from scrapy.logformatter import LogFormatter
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.throttling import ThrottlingManagerProtocol
|
||||
from scrapy.utils.request import RequestFingerprinterProtocol
|
||||
|
||||
|
||||
|
|
@ -84,6 +85,7 @@ class Crawler:
|
|||
self.stats: StatsCollector | None = None
|
||||
self.logformatter: LogFormatter | None = None
|
||||
self.request_fingerprinter: RequestFingerprinterProtocol | None = None
|
||||
self.throttler: ThrottlingManagerProtocol | None = None
|
||||
self.spider: Spider | None = None
|
||||
self.engine: ExecutionEngine | None = None
|
||||
|
||||
|
|
@ -113,6 +115,10 @@ class Crawler:
|
|||
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
|
||||
self,
|
||||
)
|
||||
self.throttler = build_from_crawler(
|
||||
load_object(self.settings["THROTTLING_MANAGER"]),
|
||||
self,
|
||||
)
|
||||
|
||||
reactor_class: str = self.settings["TWISTED_REACTOR"]
|
||||
event_loop: str = self.settings["ASYNCIO_EVENT_LOOP"]
|
||||
|
|
|
|||
|
|
@ -2,9 +2,10 @@ from __future__ import annotations
|
|||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from warnings import warn
|
||||
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
|
|
@ -24,6 +25,14 @@ class AutoThrottle:
|
|||
if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
|
||||
raise NotConfigured
|
||||
|
||||
warn(
|
||||
"You have set the AUTOTHROTTLE_ENABLED setting to True, however "
|
||||
"the AutoThrottle extension is deprecated; use throttling and "
|
||||
"backoff settings instead: "
|
||||
"https://docs.scrapy.org/en/latest/topics/throttling.html",
|
||||
ScrapyDeprecationWarning,
|
||||
)
|
||||
|
||||
self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
|
||||
self.target_concurrency: float = crawler.settings.getfloat(
|
||||
"AUTOTHROTTLE_TARGET_CONCURRENCY"
|
||||
|
|
|
|||
|
|
@ -1,87 +1,13 @@
|
|||
# Scrapy settings for $project_name project
|
||||
#
|
||||
# For simplicity, this file contains only settings considered important or
|
||||
# commonly used. You can find more settings consulting the documentation:
|
||||
#
|
||||
# https://docs.scrapy.org/en/latest/topics/settings.html
|
||||
# https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
# https://docs.scrapy.org/en/latest/topics/settings.html
|
||||
|
||||
BOT_NAME = "$project_name"
|
||||
|
||||
SPIDER_MODULES = ["$project_name.spiders"]
|
||||
NEWSPIDER_MODULE = "$project_name.spiders"
|
||||
|
||||
ADDONS = {}
|
||||
# Crawl responsibly by identifying yourself (and your website) through the
|
||||
# User-Agent header:
|
||||
#USER_AGENT = "$project_name (+https://your-domain.example)"
|
||||
|
||||
|
||||
# Crawl responsibly by identifying yourself (and your website) on the user-agent
|
||||
#USER_AGENT = "$project_name (+http://www.yourdomain.com)"
|
||||
|
||||
# Obey robots.txt rules
|
||||
ROBOTSTXT_OBEY = True
|
||||
|
||||
# Concurrency and throttling settings
|
||||
#CONCURRENT_REQUESTS = 16
|
||||
CONCURRENT_REQUESTS_PER_DOMAIN = 1
|
||||
DOWNLOAD_DELAY = 1
|
||||
|
||||
# Disable cookies (enabled by default)
|
||||
#COOKIES_ENABLED = False
|
||||
|
||||
# Disable Telnet Console (enabled by default)
|
||||
#TELNETCONSOLE_ENABLED = False
|
||||
|
||||
# Override the default request headers:
|
||||
#DEFAULT_REQUEST_HEADERS = {
|
||||
# "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
# "Accept-Language": "en",
|
||||
#}
|
||||
|
||||
# Enable or disable spider middlewares
|
||||
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
|
||||
#SPIDER_MIDDLEWARES = {
|
||||
# "$project_name.middlewares.${ProjectName}SpiderMiddleware": 543,
|
||||
#}
|
||||
|
||||
# Enable or disable downloader middlewares
|
||||
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
|
||||
#DOWNLOADER_MIDDLEWARES = {
|
||||
# "$project_name.middlewares.${ProjectName}DownloaderMiddleware": 543,
|
||||
#}
|
||||
|
||||
# Enable or disable extensions
|
||||
# See https://docs.scrapy.org/en/latest/topics/extensions.html
|
||||
#EXTENSIONS = {
|
||||
# "scrapy.extensions.telnet.TelnetConsole": None,
|
||||
#}
|
||||
|
||||
# Configure item pipelines
|
||||
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
|
||||
#ITEM_PIPELINES = {
|
||||
# "$project_name.pipelines.${ProjectName}Pipeline": 300,
|
||||
#}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
|
||||
#AUTOTHROTTLE_ENABLED = True
|
||||
# The initial download delay
|
||||
#AUTOTHROTTLE_START_DELAY = 5
|
||||
# The maximum download delay to be set in case of high latencies
|
||||
#AUTOTHROTTLE_MAX_DELAY = 60
|
||||
# The average number of requests Scrapy should be sending in parallel to
|
||||
# each remote server
|
||||
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||
# Enable showing throttling stats for every response received:
|
||||
#AUTOTHROTTLE_DEBUG = False
|
||||
|
||||
# Enable and configure HTTP caching (disabled by default)
|
||||
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
|
||||
#HTTPCACHE_ENABLED = True
|
||||
#HTTPCACHE_EXPIRATION_SECS = 0
|
||||
#HTTPCACHE_DIR = "httpcache"
|
||||
#HTTPCACHE_IGNORE_HTTP_CODES = []
|
||||
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
|
||||
|
||||
# Set settings whose default value is deprecated to a future-proof value
|
||||
# Set settings whose default value is deprecated to a future-proof value:
|
||||
FEED_EXPORT_ENCODING = "utf-8"
|
||||
|
|
|
|||
|
|
@ -1,22 +1,190 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
import datetime as dt
|
||||
from collections.abc import Awaitable, Iterable
|
||||
from datetime import UTC
|
||||
from email.utils import parsedate_to_datetime
|
||||
from functools import wraps
|
||||
from typing import TYPE_CHECKING, Any, Callable, Protocol, TypedDict, Union
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from typing_extensions import NotRequired, Self
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
|
||||
def _parse_retry_after(response: Response) -> float | None:
|
||||
value = response.headers.get("Retry-After")
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
value = value.decode("utf-8").strip()
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
if value.isdigit():
|
||||
return float(value) # seconds
|
||||
try:
|
||||
date = parsedate_to_datetime(value)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
if date.tzinfo is None:
|
||||
date = date.replace(tzinfo=UTC)
|
||||
now = dt.datetime.now(UTC)
|
||||
seconds_to_wait = (date - now).total_seconds()
|
||||
return max(0, int(seconds_to_wait)) or None
|
||||
|
||||
|
||||
def _parse_ratelimit_reset(response: Response) -> float | None:
|
||||
value = response.headers.get("RateLimit-Reset")
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
value = value.decode("utf-8").strip()
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
class BackoffScopeData(TypedDict):
|
||||
delay: NotRequired[float]
|
||||
consumed: NotRequired[float]
|
||||
remaining: NotRequired[float]
|
||||
|
||||
|
||||
ScopeID = str
|
||||
BackoffData = Union[None, ScopeID, Iterable[ScopeID], dict[ScopeID, BackoffScopeData]]
|
||||
RequestScopes = Union[None, ScopeID, Iterable[ScopeID], dict[ScopeID, float | None]]
|
||||
|
||||
|
||||
def iter_scopes(scopes: RequestScopes) -> Iterable[ScopeID]:
|
||||
if scopes is None:
|
||||
return ()
|
||||
if isinstance(scopes, str):
|
||||
return (scopes,)
|
||||
if isinstance(scopes, dict):
|
||||
return scopes.keys()
|
||||
return iter(scopes)
|
||||
|
||||
|
||||
def add_scope(
|
||||
scopes: RequestScopes,
|
||||
scope: ScopeID,
|
||||
value: float | None = None,
|
||||
/,
|
||||
) -> RequestScopes:
|
||||
"""Add *scope* to *scopes* with *value*.
|
||||
|
||||
This is a utility function to help extending the output of
|
||||
:meth:`~ThrottlingManagerProtocol.get_scopes`, e.g. in
|
||||
:class:`ThrottlingManager` subclasses.
|
||||
"""
|
||||
if value is not None:
|
||||
if not isinstance(scopes, dict):
|
||||
if scopes is None:
|
||||
scopes = {}
|
||||
elif isinstance(scopes, str):
|
||||
scopes = {scopes: None}
|
||||
elif isinstance(scopes, Iterable):
|
||||
scopes = {s: None for s in scopes}
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Invalid type ({type(scopes)}) of scopes value "
|
||||
f"{scopes!r}. Expected None, str, Iterable or dict."
|
||||
)
|
||||
if scope in scopes and not isinstance(scopes[scope], dict):
|
||||
raise TypeError(f"Scope {scope!r} has a non-dict value in {scopes!r}")
|
||||
scopes[scope] = value
|
||||
elif scopes is None:
|
||||
scopes = scope
|
||||
elif isinstance(scopes, str):
|
||||
if scopes != scope:
|
||||
scopes = {scopes, scope}
|
||||
elif isinstance(scopes, dict):
|
||||
if scope not in scopes:
|
||||
scopes[scope] = None
|
||||
elif isinstance(scopes, Iterable):
|
||||
if scope not in scopes:
|
||||
scopes = set(scopes) | {scope}
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Invalid type ({type(scopes)}) of scopes value "
|
||||
f"{scopes!r}. Expected None, str, Iterable or dict."
|
||||
)
|
||||
return scopes
|
||||
|
||||
|
||||
def update_scope_backoff(
|
||||
backoff: BackoffData,
|
||||
scope: ScopeID,
|
||||
/,
|
||||
*,
|
||||
delay: float | None = None,
|
||||
consumed: float | None = None,
|
||||
) -> BackoffData:
|
||||
"""Add *scope* to *backoff* or update its existing entry the given
|
||||
parameters.
|
||||
|
||||
This is a utility function to help extending the output of
|
||||
:meth:`~ThrottlingManagerProtocol.get_initial_backoff`,
|
||||
:meth:`~ThrottlingManagerProtocol.get_response_backoff` or
|
||||
:meth:`~ThrottlingManagerProtocol.get_exception_backoff`, e.g. in
|
||||
:class:`ThrottlingManager` subclasses.
|
||||
"""
|
||||
has_params = delay is not None or consumed is not None
|
||||
if has_params:
|
||||
if not isinstance(backoff, dict):
|
||||
if backoff is None:
|
||||
backoff = {}
|
||||
elif isinstance(backoff, str):
|
||||
backoff = {backoff: {}}
|
||||
elif isinstance(backoff, Iterable):
|
||||
backoff = {s: {} for s in backoff}
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Invalid type ({type(backoff)}) of scopes value "
|
||||
f"{backoff!r}. Expected None, str, Iterable or dict."
|
||||
)
|
||||
if scope in backoff:
|
||||
if not isinstance(backoff[scope], dict):
|
||||
raise TypeError(f"Scope {scope!r} has a non-dict value in {backoff!r}")
|
||||
else:
|
||||
backoff[scope] = {}
|
||||
if delay is not None:
|
||||
backoff[scope]["delay"] = delay
|
||||
if consumed is not None:
|
||||
backoff[scope]["consumed"] = consumed
|
||||
elif backoff is None:
|
||||
backoff = scope
|
||||
elif isinstance(backoff, str):
|
||||
if backoff != scope:
|
||||
backoff = {backoff, scope}
|
||||
elif isinstance(backoff, dict):
|
||||
if scope not in backoff:
|
||||
backoff[scope] = {}
|
||||
elif isinstance(backoff, Iterable):
|
||||
if scope not in backoff:
|
||||
backoff = set(backoff) | {scope}
|
||||
else:
|
||||
raise TypeError(
|
||||
f"Invalid type ({type(backoff)}) of scopes value "
|
||||
f"{backoff!r}. Expected None, str, Iterable or dict."
|
||||
)
|
||||
return backoff
|
||||
|
||||
|
||||
class ThrottlingManagerProtocol(Protocol):
|
||||
"""A protocol for :setting:`THROTTLING_MANAGER` :ref:`components
|
||||
<topics-components>`."""
|
||||
|
||||
def get_scopes(
|
||||
self, request: Request
|
||||
) -> None | str | Iterable[str] | dict[str, float]:
|
||||
async def get_scopes(self, request: Request) -> RequestScopes:
|
||||
"""Return the :ref:`throttling scopes <throttling-scopes>` that apply
|
||||
to *request*.
|
||||
|
||||
|
|
@ -25,29 +193,39 @@ class ThrottlingManagerProtocol(Protocol):
|
|||
keys and :ref:`throttling quotas <throttling-quotas>` as values.
|
||||
"""
|
||||
|
||||
def get_response_throttling(
|
||||
self, response: Response
|
||||
) -> None | str | Iterable[str] | dict[str, dict[str, Any]]:
|
||||
"""Return a throttling data update based on *response*.
|
||||
async def get_initial_backoff(self) -> BackoffData:
|
||||
"""Return the initial throttling data.
|
||||
|
||||
Return ``None`` if there is nothing new to report, i.e. the response is
|
||||
not a :ref:`backoff <backoff>` response.
|
||||
This method is called before the first request is sent, and it should
|
||||
be used to provide an initial throttling state, to be used before it is
|
||||
updated with later calls to :meth:`get_response_backoff` and
|
||||
:meth:`get_exception_backoff`.
|
||||
|
||||
If the response indicates that one or more scopes are currently
|
||||
exhausted, return a string for a single scope or an iterable of strings
|
||||
for multiple scopes.
|
||||
**Return values:**
|
||||
|
||||
If the response indicates any other information about one or more
|
||||
scopes, return a dict with scopes as keys and dict values. Dict values
|
||||
support the following keys:
|
||||
You may return any of the following:
|
||||
|
||||
- ``"delay"``: a float indicating how many seconds to wait before
|
||||
sending another request for the scope.
|
||||
- ``None``: no throttling data to report.
|
||||
|
||||
- ``"quota"``: a float indicating the remaining :ref:`throttling
|
||||
quota <throttling-quotas>`.
|
||||
- A string: a single scope name, indicating that the scope is
|
||||
currently exhausted.
|
||||
|
||||
If ``"quota"`` is not specified, the resource is considered exhausted.
|
||||
- An iterable of strings: multiple scope names, indicating that
|
||||
those scopes are currently exhausted.
|
||||
|
||||
- A dict with scope names as keys and dict values. Dict values
|
||||
support the following keys:
|
||||
|
||||
- ``"delay"``: a float indicating how many seconds to wait before
|
||||
sending another request for the scope.
|
||||
|
||||
- ``"quota"``: a float indicating the remaining :ref:`throttling
|
||||
quota <throttling-quotas>`.
|
||||
|
||||
If ``"quota"`` is not specified, the resource is considered
|
||||
exhausted.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
|
|
@ -58,16 +236,63 @@ class ThrottlingManagerProtocol(Protocol):
|
|||
}
|
||||
"""
|
||||
|
||||
def get_exception_throttling(
|
||||
async def get_response_backoff(self, response: Response) -> BackoffData:
|
||||
"""Return a throttling data update based on *response*.
|
||||
|
||||
It supports the same return values as :meth:`get_initial_backoff`.
|
||||
"""
|
||||
|
||||
async def get_exception_backoff(
|
||||
self, request: Request, exception: Exception
|
||||
) -> None | str | Iterable[str] | dict[str, dict[str, Any]]:
|
||||
) -> BackoffData:
|
||||
"""Return a throttling data update based on *exception* and the
|
||||
*request* that caused it.
|
||||
|
||||
It supports the same return values as :meth:`get_response_throttling`.
|
||||
It supports the same return values as :meth:`get_initial_backoff`.
|
||||
"""
|
||||
|
||||
|
||||
GetScopesMethod = Callable[
|
||||
[ThrottlingManagerProtocol, Request], Awaitable[RequestScopes]
|
||||
]
|
||||
|
||||
|
||||
def scope_cache(f: GetScopesMethod) -> GetScopesMethod:
|
||||
"""Decorator to cache the result of
|
||||
:meth:`~ThrottlingManagerProtocol.get_scopes` calls.
|
||||
|
||||
It should be used so that calls to
|
||||
:meth:`~ThrottlingManagerProtocol.get_scopes` from methods like
|
||||
:meth:`~ThrottlingManagerProtocol.get_response_backoff` or
|
||||
:meth:`~ThrottlingManagerProtocol.get_exception_backoff` do not become
|
||||
unnecessarily expensive.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.throttling import scope_cache
|
||||
|
||||
|
||||
class MyThrottlingManager:
|
||||
@scope_cache
|
||||
async def get_scopes(self, request):
|
||||
return urlparse_cached(request).netloc
|
||||
"""
|
||||
cache = WeakKeyDictionary()
|
||||
|
||||
@wraps(f)
|
||||
async def wrapper(self, request: Request):
|
||||
if request in cache:
|
||||
return cache[request]
|
||||
scopes = await f(self, request)
|
||||
cache[request] = scopes
|
||||
return scopes
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class ThrottlingManager:
|
||||
"""The default :setting:`THROTTLING_MANAGER` class.
|
||||
|
||||
|
|
@ -75,7 +300,99 @@ class ThrottlingManager:
|
|||
backoff according to :ref:`backoff settings <basic-throttling>`.
|
||||
"""
|
||||
|
||||
def get_scopes(
|
||||
self, request: Request
|
||||
) -> None | str | Iterable[str] | dict[str, float]:
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler) -> Self:
|
||||
return cls(crawler)
|
||||
|
||||
def __init__(self, crawler: Crawler) -> None:
|
||||
self.crawler = crawler
|
||||
self.throttler = crawler.throttler
|
||||
self.backoff_http_codes = set(crawler.settings.getlist("BACKOFF_HTTP_CODES"))
|
||||
self.backoff_exceptions = tuple(
|
||||
load_object(cls) for cls in crawler.settings.getlist("BACKOFF_EXCEPTIONS")
|
||||
)
|
||||
|
||||
@scope_cache
|
||||
async def get_scopes(
|
||||
self: ThrottlingManagerProtocol, request: Request
|
||||
) -> RequestScopes:
|
||||
return urlparse_cached(request).netloc
|
||||
|
||||
async def get_initial_backoff(self) -> BackoffData:
|
||||
return None
|
||||
|
||||
async def get_response_backoff(self, response: Response) -> BackoffData:
|
||||
if response.status not in self.backoff_http_codes:
|
||||
return None
|
||||
assert response.request is not None
|
||||
assert self.throttler is not None
|
||||
scopes = await self.throttler.get_scopes(response.request)
|
||||
if delay := self.get_response_delay(response):
|
||||
scopes = {scope: {"delay": delay} for scope in iter_scopes(scopes)}
|
||||
return scopes
|
||||
|
||||
def get_response_delay(self, response: Response) -> float | None:
|
||||
"""Return the throttling delay requested by the response."""
|
||||
retry_after = _parse_retry_after(response)
|
||||
ratelimit_reset = _parse_ratelimit_reset(response)
|
||||
if retry_after is None and ratelimit_reset is None:
|
||||
return None
|
||||
if retry_after is not None and ratelimit_reset is not None:
|
||||
return max(retry_after, ratelimit_reset)
|
||||
if retry_after is not None:
|
||||
return retry_after
|
||||
assert ratelimit_reset is not None
|
||||
return ratelimit_reset
|
||||
|
||||
async def get_exception_backoff(
|
||||
self, request: Request, exception: Exception
|
||||
) -> BackoffData:
|
||||
if isinstance(exception, self.backoff_exceptions):
|
||||
assert self.throttler is not None
|
||||
return await self.throttler.get_scopes(request)
|
||||
return None
|
||||
|
||||
|
||||
class ThrottlingScopeManagerProtocol(Protocol):
|
||||
"""A protocol for :setting:`THROTTLING_SCOPE_MANAGER` :ref:`components
|
||||
<topics-components>`.
|
||||
|
||||
The ``__init__`` method gets a ``config`` dict with the base configuration
|
||||
of the managed throttling scope. For example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
{
|
||||
"id": "example.com",
|
||||
"concurrency": 1.0,
|
||||
"delay": 1.0,
|
||||
"jitter": 0.5,
|
||||
"quota": 1000.0,
|
||||
"window": 60.0,
|
||||
"backoff": {
|
||||
"http_codes": [429, 503],
|
||||
"exceptions": ["builtins.IOError"],
|
||||
"delay_factor": 1.2,
|
||||
"max_delay": 180.0,
|
||||
"min_delay": 5.0,
|
||||
"jitter": [0.01, 0.33],
|
||||
"concurrency_factor": 0.8,
|
||||
},
|
||||
"rampup": {
|
||||
"backoff_target": 1,
|
||||
"delay_factor": 0.8,
|
||||
"min_delay": 0.05,
|
||||
},
|
||||
}
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler: Crawler, config: dict[str, Any]) -> Self:
|
||||
return cls(crawler, config)
|
||||
|
||||
def __init__(self, crawler: Crawler, config: dict[str, Any]) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class ThrottlingScopeManager:
|
||||
"""The default :setting:`THROTTLING_SCOPE_MANAGER` class."""
|
||||
|
|
|
|||
Loading…
Reference in New Issue