WIP

scrapy/core/downloader/__init__.py

@@ -2,8 +2,10 @@ from __future__ import annotations
 
 import random
 import warnings
+from collections import deque
 from collections.abc import Iterator, Mapping
 from dataclasses import dataclass, field
+from datetime import datetime
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet.defer import inlineCallbacks
@@ -30,15 +32,46 @@ if TYPE_CHECKING:
         ThrottlingManagerProtocol,
         ThrottlingScopeManagerProtocol,
     )
+    from scrapy.utils.asyncio import CallLaterResult
 
 
 @dataclass(slots=True, eq=False)
 class _Slot:
     """Downloader slot"""
 
+    concurrency: int
+    delay: float
+    randomize_delay: bool
+
     active: set[Request] = field(default_factory=set, init=False, repr=False)
+    queue: deque[tuple[Request, Deferred[Response]]] = field(
+        default_factory=deque, init=False, repr=False
+    )
     transferring: set[Request] = field(default_factory=set, init=False, repr=False)
     lastseen: float = field(default=0, init=False, repr=False)
+    latercall: CallLaterResult | None = field(default=None, init=False, repr=False)
+
+    def free_transfer_slots(self) -> int:
+        return self.concurrency - len(self.transferring)
+
+    def download_delay(self) -> float:
+        if self.randomize_delay:
+            return random.uniform(0.5 * self.delay, 1.5 * self.delay)  # noqa: S311
+        return self.delay
+
+    def close(self) -> None:
+        if self.latercall:
+            self.latercall.cancel()
+            self.latercall = None
+
+    def __str__(self) -> str:
+        return (
+            f"<downloader.Slot concurrency={self.concurrency!r} "
+            f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
+            f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
+            f"len(transferring)={len(self.transferring)} "
+            f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
+        )
 
 
 Slot = create_deprecated_class(
@@ -205,6 +238,26 @@ class Downloader:
     def needs_backout(self) -> bool:
         return len(self.active) >= self.total_concurrency
 
+    @property
+    def domain_concurrency(self) -> int:
+        warnings.warn(
+            "Downloader.domain_concurrency is deprecated. Per-domain concurrency "
+            "limits are now managed by the throttling system.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return self.settings.getint("CONCURRENT_REQUESTS_PER_DOMAIN")
+
+    @property
+    def randomize_delay(self) -> bool:
+        warnings.warn(
+            "Downloader.randomize_delay is deprecated. Delay randomization is now "
+            "managed by the throttling system.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
+
     @property
     def slots(self) -> _DeprecatedSlotsView:
         warnings.warn(
@@ -235,7 +288,7 @@ class Downloader:
         meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
         if meta_slot is not None:
             return meta_slot
-        return urlparse_cached(request).netloc or ""
+        return urlparse_cached(request).hostname or ""
 
     async def _enqueue_request(self, request: Request) -> Response:
         key = self._get_slot_key(request)

scrapy/throttling.py

@@ -473,7 +473,7 @@ def scope_cache(f: _GetScopesMethod) -> _GetScopesMethod:
         class MyThrottlingManager:
             @scope_cache
             async def get_scopes(self, request):
-                return urlparse_cached(request).netloc
+                return urlparse_cached(request).hostname or ""
     """
 
     @wraps(f)
@@ -542,8 +542,8 @@ class ThrottlingManager:
         :setting:`DOWNLOAD_SLOTS` setting into :setting:`THROTTLING_SCOPES`.
 
         Each ``DOWNLOAD_SLOTS`` entry is translated to a throttling scope keyed
-        by the same slot name (the default manager keys domain scopes by
-        ``netloc``, which is what download slots used too): ``concurrency`` and
+        by the same slot name (the default manager keys domain scopes by host
+        name, which is what download slots used too): ``concurrency`` and
         ``delay`` map directly, and the ``randomize_delay`` boolean maps to a
         ``jitter`` magnitude (the historical ±50%, or none). An explicit
         ``THROTTLING_SCOPES`` entry for the same scope takes precedence over the
@@ -592,7 +592,7 @@ class ThrottlingManager:
                 stacklevel=2,
             )
             return cast("RequestScopes", download_slot)
-        return urlparse_cached(request).netloc
+        return urlparse_cached(request).hostname or ""
 
     def get_slot_key(self, request: Request) -> str:
         scopes = self._resolve_scopes_sync(request)
@@ -852,7 +852,9 @@ class ThrottlingManager:
         except Exception:  # pragma: no cover - backend-specific failures
             return
         if delay:
-            self.apply_robots_crawl_delay(urlparse_cached(request).netloc, delay)
+            self.apply_robots_crawl_delay(
+                urlparse_cached(request).hostname or "", delay
+            )
 
     def apply_robots_crawl_delay(self, scope_id: ScopeID, delay: float) -> None:
         """Honor a robots.txt ``Crawl-delay`` directive of *delay* seconds for