diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 8e4053a92..7e570ace5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -2,8 +2,10 @@ from __future__ import annotations import random import warnings +from collections import deque from collections.abc import Iterator, Mapping from dataclasses import dataclass, field +from datetime import datetime from typing import TYPE_CHECKING, Any from twisted.internet.defer import inlineCallbacks @@ -30,15 +32,46 @@ if TYPE_CHECKING: ThrottlingManagerProtocol, ThrottlingScopeManagerProtocol, ) + from scrapy.utils.asyncio import CallLaterResult @dataclass(slots=True, eq=False) class _Slot: """Downloader slot""" + concurrency: int + delay: float + randomize_delay: bool + active: set[Request] = field(default_factory=set, init=False, repr=False) + queue: deque[tuple[Request, Deferred[Response]]] = field( + default_factory=deque, init=False, repr=False + ) transferring: set[Request] = field(default_factory=set, init=False, repr=False) lastseen: float = field(default=0, init=False, repr=False) + latercall: CallLaterResult | None = field(default=None, init=False, repr=False) + + def free_transfer_slots(self) -> int: + return self.concurrency - len(self.transferring) + + def download_delay(self) -> float: + if self.randomize_delay: + return random.uniform(0.5 * self.delay, 1.5 * self.delay) # noqa: S311 + return self.delay + + def close(self) -> None: + if self.latercall: + self.latercall.cancel() + self.latercall = None + + def __str__(self) -> str: + return ( + f"" + ) Slot = create_deprecated_class( @@ -205,6 +238,26 @@ class Downloader: def needs_backout(self) -> bool: return len(self.active) >= self.total_concurrency + @property + def domain_concurrency(self) -> int: + warnings.warn( + "Downloader.domain_concurrency is deprecated. Per-domain concurrency " + "limits are now managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN") + + @property + def randomize_delay(self) -> bool: + warnings.warn( + "Downloader.randomize_delay is deprecated. Delay randomization is now " + "managed by the throttling system.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + return self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") + @property def slots(self) -> _DeprecatedSlotsView: warnings.warn( @@ -235,7 +288,7 @@ class Downloader: meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT) if meta_slot is not None: return meta_slot - return urlparse_cached(request).netloc or "" + return urlparse_cached(request).hostname or "" async def _enqueue_request(self, request: Request) -> Response: key = self._get_slot_key(request) diff --git a/scrapy/throttling.py b/scrapy/throttling.py index cc38a0d47..c8a910e83 100644 --- a/scrapy/throttling.py +++ b/scrapy/throttling.py @@ -473,7 +473,7 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod: class MyThrottlingManager: @scope_cache async def get_scopes(self, request): - return urlparse_cached(request).netloc + return urlparse_cached(request).hostname or "" """ @wraps(f) @@ -542,8 +542,8 @@ class ThrottlingManager: :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLING_SCOPES`. Each ``DOWNLOAD_SLOTS`` entry is translated to a throttling scope keyed - by the same slot name (the default manager keys domain scopes by - ``netloc``, which is what download slots used too): ``concurrency`` and + by the same slot name (the default manager keys domain scopes by host + name, which is what download slots used too): ``concurrency`` and ``delay`` map directly, and the ``randomize_delay`` boolean maps to a ``jitter`` magnitude (the historical ±50%, or none). An explicit ``THROTTLING_SCOPES`` entry for the same scope takes precedence over the @@ -592,7 +592,7 @@ class ThrottlingManager: stacklevel=2, ) return cast("RequestScopes", download_slot) - return urlparse_cached(request).netloc + return urlparse_cached(request).hostname or "" def get_slot_key(self, request: Request) -> str: scopes = self._resolve_scopes_sync(request) @@ -852,7 +852,9 @@ class ThrottlingManager: except Exception: # pragma: no cover - backend-specific failures return if delay: - self.apply_robots_crawl_delay(urlparse_cached(request).netloc, delay) + self.apply_robots_crawl_delay( + urlparse_cached(request).hostname or "", delay + ) def apply_robots_crawl_delay(self, scope_id: ScopeID, delay: float) -> None: """Honor a robots.txt ``Crawl-delay`` directive of *delay* seconds for