RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Cover backward-incompatible changes in the release notes

This commit is contained in:
Adrián Chaves 2025-03-12 08:38:27 +01:00
parent f0c933f094
commit 53841ff419
30 changed files with 136 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
docs/news.rst
View File

@ -15,9 +15,22 @@ Highlights:
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- ``scrapy.core.engine.Slot.start_requests` and its matching
``Slot.__init__()`` parameter have been removed, replaced by
``seeds_iterator``.
- The second parameter of
``scrapy.core.engine.ExecutionEngine.open_spider()``, ``start_requests``,
has been removed. The starting requests are determined by the ``spider``
parameter instead.
- ``scrapy.core.spidermw.SpiderMiddlewareManager.process_start_requests()``
has been replaced by
``scrapy.core.spidermw.SpiderMiddlewareManager.process_seeds()``.
- ``scrapy.core.engine.Slot`` has been renamed to
``scrapy.core.engine._Slot`` and should not be used.
- ``scrapy.core.engine.ExecutionEngine.slot`` has been renamed to
``scrapy.core.engine.ExecutionEngine._slot`` and should not be used.
- The ``slot`` :ref:`telnet variable <telnet-vars>` has been removed.
Deprecations
~~~~~~~~~~~~

12
docs/topics/telnetconsole.rst
View File

@ -59,6 +59,8 @@ Default Username and Password can be overridden by the settings
You need the telnet program which comes installed by default in Windows, and
most Linux distros.
.. _telnet-vars:
Available variables in the telnet console
=========================================
@ -77,8 +79,6 @@ convenience:
+----------------+-------------------------------------------------------------------+
| ``spider`` | the active spider |
+----------------+-------------------------------------------------------------------+
| ``slot`` | the engine slot |
+----------------+-------------------------------------------------------------------+
| ``extensions`` | the Extension Manager (Crawler.extensions attribute) |
+----------------+-------------------------------------------------------------------+
| ``stats`` | the Stats Collector (Crawler.stats attribute) |
@ -114,10 +114,10 @@ using the telnet console::
engine.scraper.is_idle() : False
engine.spider.name : followall
engine.spider_is_idle() : False
engine.slot.closing : False
len(engine.slot.inprogress) : 16
len(engine.slot.scheduler.dqs or []) : 0
len(engine.slot.scheduler.mqs) : 92
engine._slot.closing : False
len(engine._slot.inprogress) : 16
len(engine._slot.scheduler.dqs or []) : 0
len(engine._slot.scheduler.mqs) : 92
len(engine.scraper.slot.queue) : 0
len(engine.scraper.slot.active) : 0
engine.scraper.slot.active_size : 0

4
extras/qpsclient.py
View File

@ -35,6 +35,10 @@ class QPSSpider(Spider):
self.download_delay = float(self.download_delay)
async def yield_seeds(self):
for seed in self.start_requests():
yield seed
def start_requests(self):
url = self.benchurl
if self.latency is not None:
url += f"?latency={self.latency}"

3
scrapy/commands/check.py
View File

@ -81,8 +81,7 @@ class Command(ScrapyCommand):
spider_loader = self.crawler_process.spider_loader
async def yield_seeds(self):
requests = conman.from_spider(self, self._result)
for request in requests:
for request in conman.from_spider(self, self._result):
yield request
with set_environ(SCRAPY_CHECK="true"):

80
scrapy/core/engine.py
View File

@ -42,14 +42,12 @@ logger = logging.getLogger(__name__)
_T = TypeVar("_T")
class Slot:
class _Slot:
def __init__(
self,
close_if_idle: bool,
nextcall: CallLaterOnce[Deferred[None]],
scheduler: BaseScheduler,
*,
seeds: AsyncIterator[Any] | None,
) -> None:
self.closing: Deferred[None] | None = None
self.inprogress: set[Request] = set()
@ -57,7 +55,6 @@ class Slot:
self.nextcall: CallLaterOnce[Deferred[None]] = nextcall
self.scheduler: BaseScheduler = scheduler
self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
self.seeds: AsyncIterator[Any] | None = seeds
def add_request(self, request: Request) -> None:
self.inprogress.add(request)
@ -100,7 +97,7 @@ class ExecutionEngine:
self.signals: SignalManager = crawler.signals
assert crawler.logformatter
self.logformatter: LogFormatter = crawler.logformatter
self.slot: Slot | None = None
self._slot: _Slot | None = None
self.spider: Spider | None = None
self.running: bool = False
self.paused: bool = False
@ -115,6 +112,7 @@ class ExecutionEngine:
)
self.start_time: float | None = None
self._load_seeding_policy()
self._seeds: AsyncIterator[Any] | None = None
def _load_seeding_policy(self) -> None:
try:
@ -195,16 +193,16 @@ class ExecutionEngine:
@inlineCallbacks
def _process_next_seed(self):
try:
seed = yield deferred_from_coro(self.slot.seeds.__anext__())
seed = yield deferred_from_coro(self._seeds.__anext__())
except StopAsyncIteration:
self.slot.seeds = None
self._seeds = None
except RuntimeError:
# “RuntimeError: anext(): asynchronous generator is already
# running” happens if yield_seeds is taking long to yield the
# next seed.
pass
except Exception:
self.slot.seeds = None
self._seeds = None
logger.error(
"Error while reading seeds",
exc_info=True,
@ -215,33 +213,33 @@ class ExecutionEngine:
self.crawl(seed)
else:
self.scraper.start_itemproc(seed, response=None)
self.slot.nextcall.schedule()
self._slot.nextcall.schedule()
@inlineCallbacks
def _start_next_requests(self) -> Generator[Deferred[Any], Any, None]:
if self.slot is None or self.paused:
if self._slot is None or self._slot.closing is not None or self.paused:
return
self._start_scheduled_requests()
if self.slot.seeds is not None and not self._needs_backout():
if self._seeds is not None and not self._needs_backout():
yield self._process_next_seed()
if self.spider_is_idle() and self.slot.close_if_idle:
if self.spider_is_idle() and self._slot.close_if_idle:
self._spider_idle()
def _needs_backout(self) -> bool:
assert self.slot is not None # typing
assert self._slot is not None # typing
assert self.scraper.slot is not None # typing
return (
not self.running
or bool(self.slot.closing)
or bool(self._slot.closing)
or self.downloader.needs_backout()
or self.scraper.slot.needs_backout()
)
def _start_scheduled_request(self) -> Deferred[None] | None:
assert self.slot is not None # typing
assert self._slot is not None # typing
assert self.spider is not None # typing
request = self.slot.scheduler.next_request()
request = self._slot.scheduler.next_request()
if request is None:
return None
@ -256,8 +254,8 @@ class ExecutionEngine:
)
def _remove_request(_: Any) -> None:
assert self.slot
self.slot.remove_request(request)
assert self._slot
self._slot.remove_request(request)
d2: Deferred[None] = d.addBoth(_remove_request)
d2.addErrback(
@ -267,7 +265,7 @@ class ExecutionEngine:
extra={"spider": self.spider},
)
)
slot = self.slot
slot = self._slot
d2.addBoth(lambda _: slot.nextcall.schedule())
d2.addErrback(
lambda f: logger.info(
@ -304,22 +302,22 @@ class ExecutionEngine:
return d
def spider_is_idle(self) -> bool:
if self.slot is None:
if self._slot is None:
raise RuntimeError("Engine slot not assigned")
if not self.scraper.slot.is_idle(): # type: ignore[union-attr]
return False
if self.downloader.active: # downloader has pending requests
return False
if self.slot.seeds is not None: # not all start requests are handled
if self._seeds is not None: # not all start requests are handled
return False
return not self.slot.scheduler.has_pending_requests()
return not self._slot.scheduler.has_pending_requests()
def crawl(self, request: Request) -> None:
"""Inject the request into the spider <-> downloader pipeline"""
if self.spider is None:
raise RuntimeError(f"No open spider to crawl: {request}")
self._schedule_request(request, self.spider)
self.slot.nextcall.schedule() # type: ignore[union-attr]
self._slot.nextcall.schedule() # type: ignore[union-attr]
def _schedule_request(self, request: Request, spider: Spider) -> None:
request_scheduled_result = self.signals.send_catch_log(
@ -331,7 +329,7 @@ class ExecutionEngine:
for handler, result in request_scheduled_result:
if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest):
return
if not self.slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
self.signals.send_catch_log(
signals.request_dropped, request=request, spider=spider
)
@ -348,14 +346,14 @@ class ExecutionEngine:
def _downloaded(
self, result: Response | Request | Failure, request: Request
) -> Deferred[Response] | Response | Failure:
assert self.slot is not None # typing
self.slot.remove_request(request)
assert self._slot is not None # typing
self._slot.remove_request(request)
return self.download(result) if isinstance(result, Request) else result
def _download(self, request: Request) -> Deferred[Response | Request]:
assert self.slot is not None # typing
assert self._slot is not None # typing
self.slot.add_request(request)
self._slot.add_request(request)
def _on_success(result: Response | Request) -> Response | Request:
if not isinstance(result, (Response, Request)):
@ -380,8 +378,8 @@ class ExecutionEngine:
return result
def _on_complete(_: _T) -> _T:
assert self.slot is not None
self.slot.nextcall.schedule()
assert self._slot is not None
self._slot.nextcall.schedule()
return _
assert self.spider is not None
@ -396,13 +394,13 @@ class ExecutionEngine:
spider: Spider,
close_if_idle: bool = True,
) -> Generator[Deferred[Any], Any, None]:
if self.slot is not None:
if self._slot is not None:
raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
logger.info("Spider opened", extra={"spider": spider})
nextcall = CallLaterOnce(self._start_next_requests)
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
seeds = yield self.scraper.spidermw.process_seeds(spider)
self.slot = Slot(close_if_idle, nextcall, scheduler, seeds=seeds)
self._seeds = yield self.scraper.spidermw.process_seeds(spider)
self._slot = _Slot(close_if_idle, nextcall, scheduler)
self.spider = spider
if hasattr(scheduler, "open") and (d := scheduler.open(spider)):
yield d
@ -410,8 +408,8 @@ class ExecutionEngine:
assert self.crawler.stats
self.crawler.stats.open_spider(spider)
yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
self.slot.nextcall.schedule()
self.slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL)
self._slot.nextcall.schedule()
self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL)
def _spider_idle(self) -> None:
"""
@ -440,17 +438,17 @@ class ExecutionEngine:
def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]:
"""Close (cancel) spider and clear all its outstanding requests"""
if self.slot is None:
if self._slot is None:
raise RuntimeError("Engine slot not assigned")
if self.slot.closing is not None:
return self.slot.closing
if self._slot.closing is not None:
return self._slot.closing
logger.info(
"Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
)
dfd = self.slot.close()
dfd = self._slot.close()
def log_failure(msg: str) -> Callable[[Failure], None]:
def errback(failure: Failure) -> None:
@ -466,8 +464,8 @@ class ExecutionEngine:
dfd.addBoth(lambda _: self.scraper.close_spider(spider))
dfd.addErrback(log_failure("Scraper close failure"))
if hasattr(self.slot.scheduler, "close"):
dfd.addBoth(lambda _: cast(Slot, self.slot).scheduler.close(reason))
if hasattr(self._slot.scheduler, "close"):
dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason))
dfd.addErrback(log_failure("Scheduler close failure"))
dfd.addBoth(

10
scrapy/core/spidermw.py
View File

@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
super().__init__(*middlewares)
def _check_deprecated_process_start_requests_use(
self, middlewares: list[Any]
self, middlewares: tuple[Any]
) -> None:
deprecated_middlewares = [
middleware
@ -374,9 +374,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
) -> Generator[Deferred[Any], Any, AsyncIterator[Any]]:
self._check_deprecated_start_requests_use(spider)
if self._use_start_requests:
seeds = iter(spider.start_requests())
seeds = yield self._process_chain("process_start_requests", seeds, spider)
seeds = as_async_generator(seeds)
sync_seeds = iter(spider.start_requests())
sync_seeds = yield self._process_chain(
"process_start_requests", sync_seeds, spider
)
seeds = as_async_generator(sync_seeds)
else:
seeds = yield self._iter_seeds(spider)
seeds = yield self._process_chain("process_seeds", seeds)

1
scrapy/extensions/telnet.py
View File

@ -104,7 +104,6 @@ class TelnetConsole(protocol.ServerFactory):
telnet_vars: dict[str, Any] = {
"engine": self.crawler.engine,
"spider": self.crawler.engine.spider,
"slot": self.crawler.engine.slot,
"crawler": self.crawler,
"extensions": self.crawler.extensions,
"stats": self.crawler.stats,

4
scrapy/spiders/__init__.py
View File

@ -40,7 +40,7 @@ class Spider(object_ref):
custom_settings: dict[_SettingsKeyT, Any] | None = None
#: Seed URLs. See :meth:`yield_seeds`.
start_urls: list[str] = []
start_urls: list[str]
def __init__(self, name: str | None = None, **kwargs: Any):
if name is not None:
@ -48,6 +48,8 @@ class Spider(object_ref):
elif not getattr(self, "name", None):
raise ValueError(f"{type(self).__name__} must have a name")
self.__dict__.update(kwargs)
if not hasattr(self, "start_urls"):
self.start_urls: list[str] = []
@property
def logger(self) -> SpiderLoggerAdapter:

2
scrapy/spiders/init.py
View File

@ -30,7 +30,7 @@ class InitSpider(Spider):
)
async def yield_seeds(self) -> AsyncIterator[Any]:
async for seed in super().yield_seeds():
for seed in self.start_requests():
yield seed
def start_requests(self) -> Iterable[Request]:

2
scrapy/spiders/sitemap.py
View File

@ -54,7 +54,7 @@ class SitemapSpider(Spider):
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
async def yield_seeds(self) -> AsyncIterator[Any]:
async for seed in super().yield_seeds():
for seed in self.start_requests():
yield seed
def start_requests(self) -> Iterable[Request]:

2
scrapy/templates/project/module/middlewares.py.tmpl
View File

@ -46,8 +46,6 @@ class ${ProjectName}SpiderMiddleware:
async def process_seeds(self, seeds):
# Called with the seeds from the spider yield_seeds() method or with
# the output of the maching method of an earlier spider middleware.
# Must return only requests (not items).
async for seed in seeds:
yield seed

8
scrapy/utils/engine.py
View File

@ -18,10 +18,10 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]:
"engine.scraper.is_idle()",
"engine.spider.name",
"engine.spider_is_idle()",
"engine.slot.closing",
"len(engine.slot.inprogress)",
"len(engine.slot.scheduler.dqs or [])",
"len(engine.slot.scheduler.mqs)",
"engine._slot.closing",
"len(engine._slot.inprogress)",
"len(engine._slot.scheduler.dqs or [])",
"len(engine._slot.scheduler.mqs)",
"len(engine.scraper.slot.queue)",
"len(engine.scraper.slot.active)",
"engine.scraper.slot.active_size",

3
tests/CrawlerProcess/args_settings.py
View File

@ -15,7 +15,8 @@ class NoRequestsSpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")
return []
return
yield
process = CrawlerProcess(settings={})

3
tests/CrawlerProcess/asyncio_custom_loop.py
View File

@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/asyncio_enabled_no_reactor.py
View File

@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/asyncio_enabled_reactor.py
View File

@ -9,7 +9,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
View File

@ -16,7 +16,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
View File

@ -17,7 +17,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/multi.py
View File

@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(settings={})

3
tests/CrawlerProcess/reactor_default.py
View File

@ -9,7 +9,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(settings={})

3
tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
View File

@ -9,7 +9,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/reactor_select.py
View File

@ -11,7 +11,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(settings={})

3
tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
View File

@ -18,7 +18,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
View File

@ -10,7 +10,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(

3
tests/CrawlerProcess/simple.py
View File

@ -6,7 +6,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
process = CrawlerProcess(settings={})

3
tests/CrawlerRunner/change_reactor.py
View File

@ -11,7 +11,8 @@ class NoRequestsSpider(Spider):
}
async def yield_seeds(self):
return []
return
yield
configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"})

30
tests/test_commands.py
View File

@ -672,7 +672,8 @@ class MySpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.debug("It Works!")
return []
return
yield
"""
badspider = """
@ -848,7 +849,8 @@ class MySpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
return []
return
yield
"""
args = ["-o", "example.json"]
log = self.get_log(spider_code, args=args)
@ -868,7 +870,8 @@ class MySpider(scrapy.Spider):
json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
)
)
return []
return
yield
"""
Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8")
args = ["-O", "example.json"]
@ -889,7 +892,8 @@ class MySpider(scrapy.Spider):
name = 'myspider'
async def yield_seeds(self):
return []
return
yield
"""
args = ["-o", "example1.json", "-O", "example2.json"]
log = self.get_log(spider_code, args=args)
@ -906,7 +910,8 @@ class MySpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
return []
return
yield
"""
args = ["-o", "-:json"]
log = self.get_log(spider_code, args=args)
@ -985,7 +990,8 @@ class MySpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}")
return []
return
yield
"""
args = ["-a", "foo=42"]
log = self.get_log(spider_code, args=args)
@ -1055,7 +1061,8 @@ class MySpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.debug('It works!')
return []
return
yield
"""
log = self.get_log(spider_code)
assert "[myspider] DEBUG: It works!" in log
@ -1069,7 +1076,8 @@ class MySpider(scrapy.Spider):
async def yield_seeds(self):
self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
return []
return
yield
"""
args = ["-o", "example.json"]
log = self.get_log(spider_code, args=args)
@ -1089,7 +1097,8 @@ class MySpider(scrapy.Spider):
json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
)
)
return []
return
yield
"""
Path(self.cwd, "example.json").write_text("not empty", encoding="utf-8")
args = ["-O", "example.json"]
@ -1110,7 +1119,8 @@ class MySpider(scrapy.Spider):
name = 'myspider'
async def yield_seeds(self):
return []
return
yield
"""
args = ["-o", "example1.json", "-O", "example2.json"]
log = self.get_log(spider_code, args=args)

3
tests/test_crawler.py
View File

@ -581,7 +581,8 @@ class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def yield_seeds(self):
return []
return
yield
@pytest.mark.usefixtures("reactor_pytest")

5
tests/test_engine.py
View File

@ -29,7 +29,7 @@ from twisted.trial import unittest
from twisted.web import server, static, util
from scrapy import signals
from scrapy.core.engine import ExecutionEngine, Slot
from scrapy.core.engine import ExecutionEngine, _Slot
from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import CloseSpider, IgnoreRequest
from scrapy.http import Request
@ -498,7 +498,8 @@ def test_request_scheduled_signal(caplog):
return
yield
engine.slot = Slot(None, Mock(), scheduler, seeds=seeds)
engine._seeds = seeds()
engine._slot = _Slot(False, Mock(), scheduler)
crawler.signals.connect(signal_handler, request_scheduled)
keep_request = Request("https://keep.example")
engine._schedule_request(keep_request, spider)

4
tests/test_engine_seeding.py
View File

@ -98,10 +98,10 @@ class MainTestCase(TestCase):
await maybe_deferred_to_future(twisted_sleep(sleep_seconds))
yield Request("data:,a")
await maybe_deferred_to_future(twisted_sleep(sleep_seconds))
self.crawler.engine.slot.scheduler.enqueue_request(Request("data:,b"))
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
await maybe_deferred_to_future(twisted_sleep(sleep_seconds))
yield Request("data:,c")
self.crawler.engine.slot.scheduler.stop = True
self.crawler.engine._slot.scheduler.stop = True
def parse(self, response):
pass