mirror of https://github.com/scrapy/scrapy.git
WIP
This commit is contained in:
Adrian Chaves
parent
e0cc7ed7db
commit
ba8bb9e4e9
|
|
@ -3261,7 +3261,7 @@ New features
|
|||
- Settings corresponding to :setting:`DOWNLOAD_DELAY`,
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
:setting:`RANDOMIZE_DOWNLOAD_DELAY` can now be set on a per-domain basis
|
||||
via the new :setting:`DOWNLOAD_SLOTS` setting. (:issue:`5328`)
|
||||
via the new ``DOWNLOAD_SLOTS`` setting. (:issue:`5328`)
|
||||
|
||||
- Added :meth:`.TextResponse.jmespath`, a shortcut for JMESPath selectors
|
||||
available since parsel_ 1.8.1. (:issue:`5894`, :issue:`5915`)
|
||||
|
|
@ -7448,7 +7448,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
|
|||
|
||||
- Item loaders now support nested loaders (:issue:`1467`).
|
||||
- ``FormRequest.from_response`` improvements (:issue:`1382`, :issue:`1137`).
|
||||
- Added setting :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` and improved
|
||||
- Added setting ``AUTOTHROTTLE_TARGET_CONCURRENCY`` and improved
|
||||
AutoThrottle docs (:issue:`1324`).
|
||||
- Added ``response.text`` to get body as unicode (:issue:`1730`).
|
||||
- Anonymous S3 connections (:issue:`1358`).
|
||||
|
|
@ -8651,7 +8651,7 @@ Scrapy changes:
|
|||
|
||||
- added :ref:`topics-contracts`, a mechanism for testing spiders in a formal/reproducible way
|
||||
- added options ``-o`` and ``-t`` to the :command:`runspider` command
|
||||
- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED`
|
||||
- documented ``scrapy.extensions.throttle.AutoThrottle`` and added to extensions installed by default. You still need to enable it with ``AUTOTHROTTLE_ENABLED``
|
||||
- major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals.
|
||||
- added a ``process_start_requests()`` method to spider middlewares
|
||||
- dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info.
|
||||
|
|
|
|||
|
|
@ -882,6 +882,46 @@ Default: ``True``
|
|||
|
||||
Whether to enable downloader stats collection.
|
||||
|
||||
.. setting:: DOWNLOAD_BIND_ADDRESS
|
||||
|
||||
DOWNLOAD_BIND_ADDRESS
|
||||
---------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The default local outgoing address for download-handler connections.
|
||||
|
||||
This setting can be either:
|
||||
|
||||
- a host address as a string (e.g. ``"127.0.0.2"``), in which case the local
|
||||
port is chosen automatically, or
|
||||
|
||||
- a ``(host, port)`` tuple (e.g. ``("127.0.0.2", 50000)``) to bind to both a
|
||||
specific local interface and a specific local port.
|
||||
|
||||
For example:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Bind to this local address
|
||||
DOWNLOAD_BIND_ADDRESS = "127.0.0.2"
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
# Bind to this local address and local port
|
||||
DOWNLOAD_BIND_ADDRESS = ("127.0.0.2", 5000)
|
||||
|
||||
If set, built-in HTTP download handlers use this value by default.
|
||||
Set the :reqmeta:`bindaddress` request meta key to override it for a specific
|
||||
request.
|
||||
|
||||
.. note::
|
||||
|
||||
Handling of this setting needs to be implemented inside the :ref:`download
|
||||
handler <topics-download-handlers>`, so it's not guaranteed to be supported
|
||||
by all 3rd-party handlers. Specifying the port is unsupported by
|
||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
|
||||
|
||||
.. setting:: DOWNLOAD_HANDLERS
|
||||
|
||||
DOWNLOAD_HANDLERS
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ throttling limits, as do ``toscrape.com`` and ``books.toscrape.com``.
|
|||
|
||||
The main throttling :ref:`settings <topics-settings>` are:
|
||||
|
||||
- .. setting:: THROTTLING_SCOPE_CONCURRENCY
|
||||
- .. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1`` (:ref:`fallback <default-settings>`: ``8``))
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (default: ``8``)
|
||||
|
||||
Maximum number of simultaneous requests per domain.
|
||||
|
||||
|
|
@ -49,7 +49,7 @@ The main throttling :ref:`settings <topics-settings>` are:
|
|||
|
||||
When configuring these settings, note that:
|
||||
|
||||
- :setting:`CONCURRENT_REQUESTS` caps :setting:`THROTTLING_SCOPE_CONCURRENCY`.
|
||||
- :setting:`CONCURRENT_REQUESTS` caps :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`.
|
||||
|
||||
- If ``DOWNLOAD_DELAY`` ≥ response time, concurrency is effectively ``1``,
|
||||
because the next request to the domain is not sent until the delay elapses,
|
||||
|
|
@ -277,8 +277,8 @@ to wait between requests.
|
|||
|
||||
If :setting:`ROBOTSTXT_OBEY` and :setting:`THROTTLING_ROBOTSTXT_OBEY` are
|
||||
``True`` (default), valid ``Crawl-Delay`` directives override
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` and :setting:`DOWNLOAD_DELAY`. Concurrency
|
||||
is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`.
|
||||
Concurrency is set to ``1`` and delay is set to the value of ``Crawl-Delay``, capped at
|
||||
:setting:`THROTTLING_ROBOTSTXT_MAX_DELAY` (default: ``60.0``).
|
||||
|
||||
If :setting:`THROTTLING_SCOPES` defines a different concurrency or delay, it
|
||||
|
|
@ -440,8 +440,10 @@ Its keys are scope names and its values are
|
|||
following keys:
|
||||
|
||||
``concurrency`` (:class:`int`)
|
||||
Maximum number of concurrent requests for the scope. When unset,
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` applies instead.
|
||||
Maximum number of concurrent requests for the scope. When unset, the
|
||||
default depends on the kind of scope: :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
|
||||
for domain scopes, :setting:`CONCURRENT_REQUESTS_PER_IP` for IP scopes, and
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` for any other scope.
|
||||
|
||||
``min_concurrency`` (:class:`int`)
|
||||
Concurrency floor that :ref:`backoff <backoff>` and :ref:`rampup <rampup>`
|
||||
|
|
@ -452,7 +454,10 @@ following keys:
|
|||
:setting:`DOWNLOAD_DELAY`.
|
||||
|
||||
``jitter`` (:class:`float` or 2-:class:`list`)
|
||||
Per-scope override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`.
|
||||
Magnitude of the random variation applied to ``delay``; the per-scope
|
||||
override of :setting:`RANDOMIZE_DOWNLOAD_DELAY`. ``0`` disables it, ``0.5``
|
||||
means ±50% (the default when :setting:`RANDOMIZE_DOWNLOAD_DELAY` is on), and
|
||||
a ``[low, high]`` pair multiplies the delay by ``1 + uniform(low, high)``.
|
||||
|
||||
``quota`` (:class:`float`)
|
||||
Maximum :ref:`quota <throttling-quotas>` consumed per ``window``.
|
||||
|
|
@ -924,30 +929,39 @@ window (:setting:`BACKOFF_WINDOW`). You can use :ref:`throttling quotas
|
|||
Per-IP concurrency limiting
|
||||
---------------------------
|
||||
|
||||
A concurrency limit keyed by IP is just a throttling scope whose id is the
|
||||
request's IP, with a ``concurrency`` limit. A request then carries two scopes,
|
||||
its domain and its IP, and is only sent when **both** allow it (see
|
||||
:ref:`multiple-throttling-scopes`).
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_IP
|
||||
|
||||
- Implement a :ref:`throttling manager <custom-throttling-scopes>` that adds
|
||||
the request's IP as a second scope:
|
||||
A concurrency limit keyed by IP is a throttling scope whose id is the request's
|
||||
IP. A request then carries two scopes, its domain and its IP, and is only sent
|
||||
when **both** allow it (see :ref:`multiple-throttling-scopes`).
|
||||
|
||||
.. code-block:: python
|
||||
Set :setting:`CONCURRENT_REQUESTS_PER_IP` (default: ``0``, disabled) to a
|
||||
positive number to enable this built in: the throttler adds the IP of each
|
||||
request, resolved from the DNS cache, as a second scope, limited to that many
|
||||
concurrent requests. IP scopes whose ``concurrency`` is left unset default to
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP`.
|
||||
|
||||
import socket
|
||||
For finer control (e.g. resolving IPs that are not in the DNS cache yet, or
|
||||
grouping several hosts under one address), implement a :ref:`throttling manager
|
||||
<custom-throttling-scopes>` that adds the request's IP as a second scope
|
||||
instead:
|
||||
|
||||
from scrapy.throttling import ThrottlingManager, add_scope, scope_cache
|
||||
from scrapy.utils.asyncio import run_in_thread
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
.. code-block:: python
|
||||
|
||||
import socket
|
||||
|
||||
from scrapy.throttling import ThrottlingManager, add_scope, scope_cache
|
||||
from scrapy.utils.asyncio import run_in_thread
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
|
||||
class IPThrottlingManager(ThrottlingManager):
|
||||
@scope_cache
|
||||
async def get_scopes(self, request):
|
||||
scopes = await super().get_scopes(request)
|
||||
host = urlparse_cached(request).hostname
|
||||
address = await run_in_thread(socket.gethostbyname, host)
|
||||
return add_scope(scopes, address)
|
||||
class IPThrottlingManager(ThrottlingManager):
|
||||
@scope_cache
|
||||
async def get_scopes(self, request):
|
||||
scopes = await super().get_scopes(request)
|
||||
host = urlparse_cached(request).hostname
|
||||
address = await run_in_thread(socket.gethostbyname, host)
|
||||
return add_scope(scopes, address)
|
||||
|
||||
.. _throttling-settings:
|
||||
|
||||
|
|
@ -1033,6 +1047,18 @@ Additional settings
|
|||
Whether to log :ref:`throttling <throttling>` decisions (per-scope delays,
|
||||
backoff steps and recoveries) for debugging.
|
||||
|
||||
- .. setting:: THROTTLING_SCOPE_CONCURRENCY
|
||||
|
||||
:setting:`THROTTLING_SCOPE_CONCURRENCY` (default: ``1``)
|
||||
|
||||
Default maximum number of concurrent requests for :ref:`throttling scopes
|
||||
<throttling-scopes>` that are neither domains nor IPs, e.g. custom scopes
|
||||
added by a :ref:`custom throttling manager <custom-throttling-scopes>`.
|
||||
|
||||
Domain and IP scopes use :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
|
||||
:setting:`CONCURRENT_REQUESTS_PER_IP` instead. A scope ``concurrency`` set
|
||||
in :setting:`THROTTLING_SCOPES` overrides this.
|
||||
|
||||
- .. setting:: THROTTLING_SCOPE_LIMIT
|
||||
|
||||
:setting:`THROTTLING_SCOPE_LIMIT` (default: ``100000``)
|
||||
|
|
@ -1068,7 +1094,7 @@ The ``AutoThrottle`` extension is deprecated in favor of the throttling and
|
|||
:ref:`backoff <backoff>` system described here, which is always active and does
|
||||
not need to be enabled.
|
||||
|
||||
Setting :setting:`AUTOTHROTTLE_ENABLED` to ``True`` still works but logs a
|
||||
Setting ``AUTOTHROTTLE_ENABLED`` to ``True`` still works but logs a
|
||||
deprecation warning. To migrate, drop the ``AUTOTHROTTLE_*`` settings and use
|
||||
the following equivalents:
|
||||
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ class _DeprecatedSlotView:
|
|||
@property
|
||||
def randomize_delay(self) -> bool:
|
||||
if self._scope is not None:
|
||||
return self._scope._randomize # type: ignore[union-attr]
|
||||
return bool(self._scope._jitter) # type: ignore[union-attr]
|
||||
return False
|
||||
|
||||
@property
|
||||
|
|
|
|||
|
|
@ -92,6 +92,20 @@ class BackoffConfig(TypedDict, total=False):
|
|||
jitter: float | list[float]
|
||||
|
||||
|
||||
class RampupConfig(TypedDict, total=False):
|
||||
"""Per-scope override of the rampup settings.
|
||||
|
||||
Used as the value of the ``"rampup"`` key of :class:`ThrottlingScopeConfig`
|
||||
entries when fine-tuning rampup beyond a plain ``True``. Any key left out
|
||||
falls back to its default (or, for ``backoff_target``, to
|
||||
:setting:`RAMPUP_BACKOFF_TARGET`).
|
||||
"""
|
||||
|
||||
backoff_target: float | list[float]
|
||||
delay_factor: float
|
||||
min_delay: float
|
||||
|
||||
|
||||
class ThrottlingScopeConfig(TypedDict, total=False):
|
||||
"""Accepted keys of :setting:`THROTTLING_SCOPES` entries.
|
||||
|
||||
|
|
@ -105,10 +119,15 @@ class ThrottlingScopeConfig(TypedDict, total=False):
|
|||
"""Floor. Never drop below this during backoff/rampup."""
|
||||
|
||||
delay: float
|
||||
|
||||
jitter: float | list[float]
|
||||
"""Magnitude of the random variation applied to ``delay``; the per-scope
|
||||
override of :setting:`RANDOMIZE_DOWNLOAD_DELAY` (``0`` disables it, ``0.5``
|
||||
means ±50%)."""
|
||||
|
||||
quota: float
|
||||
window: float
|
||||
rampup: bool
|
||||
rampup: bool | RampupConfig
|
||||
|
||||
manager: str | type
|
||||
"""Import path or class of a custom :setting:`THROTTLING_SCOPE_MANAGER` for
|
||||
|
|
@ -116,6 +135,10 @@ class ThrottlingScopeConfig(TypedDict, total=False):
|
|||
|
||||
backoff: BackoffConfig
|
||||
|
||||
ignore_robots_txt: bool
|
||||
"""Silence the warning logged when this configuration is more aggressive
|
||||
than a robots.txt ``Crawl-delay``."""
|
||||
|
||||
|
||||
ScopeID = str
|
||||
BackoffData = None | ScopeID | Iterable[ScopeID] | dict[ScopeID, BackoffScopeData]
|
||||
|
|
@ -1199,8 +1222,11 @@ class ThrottlingScopeManager:
|
|||
self._base_delay: float = float(
|
||||
config.get("delay", settings.getfloat("DOWNLOAD_DELAY"))
|
||||
)
|
||||
self._randomize: bool = bool(
|
||||
config.get("randomize_delay", settings.getbool("RANDOMIZE_DOWNLOAD_DELAY"))
|
||||
# Magnitude of the random variation applied to the (non-backoff) delay.
|
||||
# Defaults to RANDOMIZE_DOWNLOAD_DELAY's historical ±50% when delay
|
||||
# randomization is on, or to no variation when it is off.
|
||||
self._jitter: float | list[float] = config.get(
|
||||
"jitter", 0.5 if settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") else 0.0
|
||||
)
|
||||
self._delay_factor: float = float(
|
||||
backoff.get("delay_factor", settings.getfloat("BACKOFF_DELAY_FACTOR"))
|
||||
|
|
@ -1211,7 +1237,7 @@ class ThrottlingScopeManager:
|
|||
self._min_delay: float = float(
|
||||
backoff.get("min_delay", settings.getfloat("BACKOFF_MIN_DELAY"))
|
||||
)
|
||||
self._jitter: float | list[float] = backoff.get(
|
||||
self._backoff_jitter: float | list[float] = backoff.get(
|
||||
"jitter", settings.getfloat("BACKOFF_JITTER")
|
||||
)
|
||||
self._window: float = settings.getfloat("BACKOFF_WINDOW")
|
||||
|
|
@ -1272,17 +1298,18 @@ class ThrottlingScopeManager:
|
|||
return float(value[0]), float(value[1])
|
||||
return float(value), float(value)
|
||||
|
||||
def _apply_jitter(self, value: float) -> float:
|
||||
if isinstance(self._jitter, (list, tuple)):
|
||||
low, high = self._jitter[0], self._jitter[1]
|
||||
@staticmethod
|
||||
def _apply_jitter(value: float, jitter: float | list[float]) -> float:
|
||||
if isinstance(jitter, (list, tuple)):
|
||||
low, high = jitter[0], jitter[1]
|
||||
return value * (1 + random.uniform(low, high)) # noqa: S311
|
||||
if not self._jitter:
|
||||
if not jitter:
|
||||
return value
|
||||
return value * random.uniform(1 - self._jitter, 1 + self._jitter) # noqa: S311
|
||||
return value * random.uniform(1 - jitter, 1 + jitter) # noqa: S311
|
||||
|
||||
def _effective_delay(self) -> float:
|
||||
if self._backoff_level == 0 and self._randomize and self._delay > 0:
|
||||
return random.uniform(0.5 * self._delay, 1.5 * self._delay) # noqa: S311
|
||||
if self._backoff_level == 0 and self._delay > 0:
|
||||
return self._apply_jitter(self._delay, self._jitter)
|
||||
return self._delay
|
||||
|
||||
def _recover(self, now: float) -> None:
|
||||
|
|
@ -1447,7 +1474,9 @@ class ThrottlingScopeManager:
|
|||
)
|
||||
grown = max(self._min_delay, grown)
|
||||
grown = min(grown, self._max_delay)
|
||||
self._delay = min(self._apply_jitter(grown), self._max_delay)
|
||||
self._delay = min(
|
||||
self._apply_jitter(grown, self._backoff_jitter), self._max_delay
|
||||
)
|
||||
self._next_allowed_time = now + self._delay
|
||||
|
||||
def reconcile_quota(
|
||||
|
|
|
|||
|
|
@ -66,13 +66,13 @@ def get_crawler(
|
|||
will be used to populate the crawler settings with a project level
|
||||
priority.
|
||||
"""
|
||||
# When needed, useful settings can be added here, e.g. ones that prevent
|
||||
# deprecation warnings (see prevent_warnings).
|
||||
settings: dict[str, Any] = {
|
||||
"TELNETCONSOLE_ENABLED": False,
|
||||
**get_reactor_settings(),
|
||||
**(settings_dict or {}),
|
||||
}
|
||||
if prevent_warnings:
|
||||
settings.setdefault("THROTTLING_SCOPE_CONCURRENCY", 8)
|
||||
runner: CrawlerRunnerBase
|
||||
if is_reactor_installed():
|
||||
runner = CrawlerRunner(settings)
|
||||
|
|
|
|||
|
|
@ -1106,7 +1106,7 @@ class TestThrottlingScopeManagerEdges:
|
|||
config={"id": "x", "backoff": {"jitter": [0.0, 0.0]}, "min_delay": 1.0}
|
||||
)
|
||||
# A [low, high] jitter range of [0, 0] leaves the value unchanged.
|
||||
assert scope._apply_jitter(4.0) == pytest.approx(4.0)
|
||||
assert scope._apply_jitter(4.0, scope._backoff_jitter) == pytest.approx(4.0)
|
||||
|
||||
def test_effective_delay_randomized(self):
|
||||
scope = _scope_manager(
|
||||
|
|
|
|||
Loading…
Reference in New Issue