mirror of https://github.com/scrapy/scrapy.git
36 lines
931 B
Python
36 lines
931 B
Python
from __future__ import annotations
|
|
|
|
from typing import TYPE_CHECKING, Any, NamedTuple, cast
|
|
|
|
from scrapy.core.downloader import Downloader
|
|
from scrapy.utils.httpobj import urlparse_cached
|
|
|
|
if TYPE_CHECKING:
|
|
from scrapy.http import Request
|
|
|
|
|
|
class MockSlot(NamedTuple):
|
|
active: list[Any]
|
|
|
|
|
|
class MockDownloader:
|
|
def __init__(self) -> None:
|
|
self.slots: dict[str, MockSlot] = {}
|
|
|
|
def get_slot_key(self, request: Request) -> str:
|
|
if Downloader.DOWNLOAD_SLOT in request.meta:
|
|
return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
|
|
|
|
return urlparse_cached(request).hostname or ""
|
|
|
|
def increment(self, slot_key: str) -> None:
|
|
slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
|
|
slot.active.append(1)
|
|
|
|
def decrement(self, slot_key: str) -> None:
|
|
slot = self.slots[slot_key]
|
|
slot.active.pop()
|
|
|
|
def close(self) -> None:
|
|
pass
|