RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

WIP

This commit is contained in:
Adrian Chaves 2026-06-25 13:38:55 +02:00
parent c93e430ccb
commit 7fdee93e42
10 changed files with 166 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/conf.py
View File

@ -145,7 +145,6 @@ coverage_ignore_pyobjects = [
# -- Options for the autodoc extension ----------------------------------------
autodoc_type_aliases = {
"BackoffData": "BackoffData",
"GetScopesMethod": "GetScopesMethod",
"RequestScopes": "RequestScopes",
}

19
docs/topics/signals.rst
View File

@ -502,6 +502,25 @@ headers_received
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
robots_parsed
~~~~~~~~~~~~~
.. signal:: robots_parsed
.. function:: robots_parsed(robotparser, request)
Sent by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
downloads and parses a ``robots.txt`` file, for the host that *request*
targets.
This signal supports :ref:`asynchronous handlers <signal-deferred>`.
:param robotparser: the parser holding the parsed ``robots.txt`` contents
:type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
:param request: the request that triggered the ``robots.txt`` download
:type request: :class:`~scrapy.Request` object
Response signals
----------------

2
scrapy/core/downloader/__init__.py
View File

@ -86,6 +86,8 @@ def _get_concurrency_delay(
delay: float = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
delay = spider.download_delay
if settings.get("DOWNLOAD_DELAY_PER_SLOT") is not None:
delay = settings.getfloat("DOWNLOAD_DELAY_PER_SLOT")
if hasattr(spider, "max_concurrent_requests"): # pragma: no cover
warn_on_deprecated_spider_attribute(

72
scrapy/core/engine.py
View File

@ -129,6 +129,13 @@ class ExecutionEngine:
self._start_request_processing_awaitable: (
asyncio.Future[None] | Deferred[None] | None
) = None
# Requests currently held by the throttling manager, waiting for their
# scopes to allow them through to the downloader.
self._throttling_waiting: set[Request] = set()
self._delayed_requests_warn_threshold: int = self.settings.getint(
"DELAYED_REQUESTS_WARN_THRESHOLD"
)
self._delayed_requests_warned: bool = False
downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"])
try:
self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class(
@ -351,6 +358,35 @@ class ExecutionEngine:
or bool(self._slot.closing)
or self.downloader.needs_backout()
or self.scraper.slot.needs_backout()
or self._throttling_needs_backout()
)
def _throttling_needs_backout(self) -> bool:
"""Apply backpressure while requests are held by the throttling manager.
Requests waiting on throttling do not occupy a downloader concurrency
slot, so without this check the engine would keep draining the
scheduler into pending throttling waits. Count them together with the
in-flight requests against the global concurrency limit.
"""
if not self._throttling_waiting:
return False
return (
len(self._throttling_waiting) + len(self.downloader.active)
>= self.downloader.total_concurrency
)
def _maybe_warn_delayed_requests(self) -> None:
if self._delayed_requests_warned:
return
if len(self._throttling_waiting) < self._delayed_requests_warn_threshold:
return
self._delayed_requests_warned = True
logger.warning(
f"There are {len(self._throttling_waiting)} requests held back by throttling. This may "
"indicate a throttling configuration that is too aggressive or a "
"site that is rate-limiting heavily. See DELAYED_REQUESTS_WARN_THRESHOLD.",
extra={"spider": self.spider},
)
def _remove_request(self, _: Any, request: Request) -> None:
@ -426,6 +462,8 @@ class ExecutionEngine:
return False
if self.downloader.active: # downloader has pending requests
return False
if self._throttling_waiting: # requests held by the throttling manager
return False
if self._start is not None: # not all start requests are handled
return False
return not self._slot.scheduler.has_pending_requests()
@ -481,6 +519,21 @@ class ExecutionEngine:
return await self.download_async(response_or_request)
return response_or_request
@inlineCallbacks
def _acquire_throttling(
self, request: Request
) -> Generator[Deferred[Any], Any, None]:
"""Wait at the throttling gate before *request* is sent, tracking it as
held meanwhile."""
self._throttling_waiting.add(request)
self._maybe_warn_delayed_requests()
throttler = self.crawler.throttler
assert throttler is not None
try:
yield deferred_from_coro(throttler.acquire(request))
finally:
self._throttling_waiting.discard(request)
@inlineCallbacks
def _download(
self, request: Request
@ -489,12 +542,19 @@ class ExecutionEngine:
assert self.spider is not None
self._slot.add_request(request)
throttler = self.crawler.throttler
assert throttler is not None
try:
result: Response | Request
if self._downloader_fetch_needs_spider:
result = yield self.downloader.fetch(request, self.spider)
else:
result = yield self.downloader.fetch(request)
yield self._acquire_throttling(request)
try:
result: Response | Request
if self._downloader_fetch_needs_spider:
result = yield self.downloader.fetch(request, self.spider)
else:
result = yield self.downloader.fetch(request)
except Exception as exc:
yield deferred_from_coro(throttler.process_exception(request, exc))
raise
if not isinstance(result, (Response, Request)):
raise TypeError(
f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}"
@ -502,6 +562,7 @@ class ExecutionEngine:
if isinstance(result, Response):
if result.request is None:
result.request = request
yield deferred_from_coro(throttler.process_response(result))
logkws = self.logformatter.crawled(result.request, result, self.spider)
if logkws is not None:
logger.log(
@ -515,6 +576,7 @@ class ExecutionEngine:
)
return result
finally:
throttler.release(request)
self._slot.nextcall.schedule()
def open_spider(

12
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
@ -98,7 +99,7 @@ class RobotsTxtMiddleware:
assert self.crawler.stats
try:
resp = await self.crawler.engine.download_async(robotsreq)
self._parse_robots(resp, netloc)
await self._parse_robots(resp, netloc, request)
except Exception as e:
if not isinstance(e, IgnoreRequest):
logger.error(
@ -115,13 +116,20 @@ class RobotsTxtMiddleware:
return await maybe_deferred_to_future(parser)
return parser
def _parse_robots(self, response: Response, netloc: str) -> None:
async def _parse_robots(
self, response: Response, netloc: str, request: Request
) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
robotparser=rp,
request=request,
)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp

1
scrapy/extensions/throttle.py
View File

@ -31,6 +31,7 @@ class AutoThrottle:
"backoff settings instead: "
"https://docs.scrapy.org/en/latest/topics/throttling.html",
ScrapyDeprecationWarning,
stacklevel=2,
)
self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")

18
scrapy/robotstxt.py
View File

@ -67,6 +67,16 @@ class RobotParser(metaclass=ABCMeta):
:type user_agent: str or bytes
"""
def crawl_delay(self, user_agent: str | bytes) -> float | None:
"""Return the ``Crawl-delay`` directive for ``user_agent`` as a number
of seconds, or ``None`` if it is not set or the backend does not support
it.
:param user_agent: User agent
:type user_agent: str or bytes
"""
return None
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -85,6 +95,10 @@ class PythonRobotParser(RobotParser):
url = to_unicode(url)
return self.rp.can_fetch(user_agent, url)
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -121,3 +135,7 @@ class ProtegoRobotParser(RobotParser):
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.can_fetch(url, user_agent)
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)

30
scrapy/settings/default_settings.py
View File

@ -235,6 +235,19 @@ AWS_SESSION_TOKEN = None
AWS_USE_SSL = None
AWS_VERIFY = None
BACKOFF_DELAY_FACTOR = 2.0
BACKOFF_EXCEPTIONS = [
"scrapy.exceptions.DownloadFailedError",
"scrapy.exceptions.DownloadTimeoutError",
"scrapy.exceptions.ResponseDataLossError",
]
BACKOFF_HTTP_CODES = [429, 502, 503, 504, 520, 521, 522, 523, 524]
BACKOFF_JITTER = 0.1
BACKOFF_MAX_DELAY = 300.0
BACKOFF_MIN_DELAY = 1.0
BACKOFF_WINDOW = 60.0
BOT_NAME = "scrapybot"
CLOSESPIDER_ERRORCOUNT = 0
@ -267,6 +280,8 @@ DEFAULT_REQUEST_HEADERS = {
"Accept-Language": "en",
}
DELAYED_REQUESTS_WARN_THRESHOLD = 500
DEPTH_LIMIT = 0
DEPTH_PRIORITY = 0
DEPTH_STATS_VERBOSE = False
@ -279,6 +294,7 @@ DNS_TIMEOUT = 60
DOWNLOAD_BIND_ADDRESS = None
DOWNLOAD_DELAY = 0
DOWNLOAD_DELAY_PER_SLOT = None
DOWNLOAD_FAIL_ON_DATALOSS = True
@ -488,6 +504,8 @@ PERIODIC_LOG_DELTA = None
PERIODIC_LOG_STATS = None
PERIODIC_LOG_TIMING_ENABLED = False
RAMPUP_BACKOFF_TARGET = 1
RANDOMIZE_DOWNLOAD_DELAY = True
REACTOR_THREADPOOL_MAXSIZE = 10
@ -566,6 +584,8 @@ STATS_DUMP = True
STATSMAILER_RCPTS = []
TARGET_RPM = None
TELNETCONSOLE_ENABLED = 1
TELNETCONSOLE_HOST = "127.0.0.1"
TELNETCONSOLE_PORT = [6023, 6073]
@ -574,9 +594,19 @@ TELNETCONSOLE_PASSWORD = None
TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager"
THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager"
THROTTLING_SCOPES = {}
THROTTLING_WINDOW = 60.0
THROTTLING_ROBOTSTXT_OBEY = True
THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0
THROTTLING_SCOPE_MAX_IDLE = 3600.0
THROTTLING_DEBUG = False
TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"
TWISTED_REACTOR_ENABLED = True
TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
URLLENGTH_LIMIT = 2083

1
scrapy/signals.py
View File

@ -21,6 +21,7 @@ response_received = object()
response_downloaded = object()
headers_received = object()
bytes_received = object()
robots_parsed = object()
item_scraped = object()
item_dropped = object()
item_error = object()

18
tests/test_downloadermiddleware_robotstxt.py
View File

@ -8,6 +8,7 @@ import pytest
from twisted.internet.defer import Deferred, DeferredList
from twisted.python import failure
from scrapy import signals
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware:
self.crawler: mock.MagicMock = mock.MagicMock()
self.crawler.settings = Settings()
self.crawler.engine.download_async = mock.AsyncMock()
self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])
def teardown_method(self):
del self.crawler
@ -74,6 +76,22 @@ Disallow: /some/randome/page.html
Request("http://site.local/wiki/Käyttäjä:"), middleware
)
@coroutine_test
async def test_robotstxt_emits_robots_parsed_signal(self):
crawler = self._get_successful_crawler()
middleware = RobotsTxtMiddleware(crawler)
request = Request("http://site.local/allowed")
await self.assertNotIgnored(request, middleware)
calls = [
kwargs
for _, kwargs in crawler.signals.send_catch_log_async.call_args_list
if kwargs.get("signal") is signals.robots_parsed
]
assert len(calls) == 1
assert calls[0]["request"] is request
assert calls[0]["robotparser"] is not None
assert "crawler" not in calls[0]
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())