RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename the recommended name for the process_start 1st parameter (seeds → start)

This commit is contained in:
Adrián Chaves 2025-03-19 20:53:02 +01:00
parent f91723628a
commit 07b6fafdd0
11 changed files with 82 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
docs/topics/spider-middleware.rst
View File

@ -70,7 +70,7 @@ one or more of these methods:
.. class:: SpiderMiddleware
.. method:: process_start(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any]
.. method:: process_start(start: AsyncIterable[Any], /) -> AsyncIterable[Any]
:async:
Iterate over the output of :meth:`~scrapy.Spider.start` or that
@ -79,12 +79,12 @@ one or more of these methods:
.. code-block:: python
async def process_start(self, seeds):
async for seed in seeds:
yield seed
async def process_start(self, start):
async for item_or_request in start:
yield item_or_request
You may yield :class:`~scrapy.Request` or :ref:`item <topics-items>`
objects, same as :meth:`~scrapy.Spider.start`, from *seeds* or
objects, same as :meth:`~scrapy.Spider.start`, from *start* or
not.
To write spider middlewares that work on Scrapy versions lower than
@ -93,8 +93,8 @@ one or more of these methods:
.. code-block:: python
def process_start_requests(self, seeds, spider):
yield from seeds
def process_start_requests(self, start, spider):
yield from start
.. method:: process_spider_input(response, spider)

16
scrapy/core/engine.py
View File

@ -103,7 +103,7 @@ class ExecutionEngine:
spider_closed_callback
)
self.start_time: float | None = None
self._seeds: AsyncIterable[Any] | None = None
self._start: AsyncIterable[Any] | None = None
self._waiting_for_seed: bool = False
def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]:
@ -177,13 +177,13 @@ class ExecutionEngine:
return
self._waiting_for_seed = True
try:
seed = yield deferred_from_coro(self._seeds.__anext__())
seed = yield deferred_from_coro(self._start.__anext__())
except StopAsyncIteration:
self._seeds = None
self._start = None
except Exception:
self._seeds = None
self._start = None
logger.error(
"Error while reading seeds",
"Error while reading start items and requests",
exc_info=True,
extra={"spider": self.spider},
)
@ -201,7 +201,7 @@ class ExecutionEngine:
if self._slot is None or self._slot.closing is not None or self.paused:
return
self._start_scheduled_requests()
if self._seeds is not None and not self._needs_backout():
if self._start is not None and not self._needs_backout():
yield self._process_next_seed()
if self.spider_is_idle() and self._slot.close_if_idle:
self._spider_idle()
@ -289,7 +289,7 @@ class ExecutionEngine:
return False
if self.downloader.active: # downloader has pending requests
return False
if self._seeds is not None: # not all start requests are handled
if self._start is not None: # not all start requests are handled
return False
return not self._slot.scheduler.has_pending_requests()
@ -380,7 +380,7 @@ class ExecutionEngine:
logger.info("Spider opened", extra={"spider": spider})
nextcall = CallLaterOnce(self._start_next_requests)
scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
self._seeds = yield self.scraper.spidermw.process_start(spider)
self._start = yield self.scraper.spidermw.process_start(spider)
self._slot = _Slot(close_if_idle, nextcall, scheduler)
self.spider = spider
if hasattr(scheduler, "open") and (d := scheduler.open(spider)):

14
scrapy/core/spidermw.py
View File

@ -394,15 +394,15 @@ class SpiderMiddlewareManager(MiddlewareManager):
) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]:
self._check_deprecated_start_requests_use(spider)
if self._use_start_requests:
sync_seeds = iter(spider.start_requests())
sync_seeds = yield self._process_chain(
"process_start_requests", sync_seeds, spider
sync_start = iter(spider.start_requests())
sync_start = yield self._process_chain(
"process_start_requests", sync_start, spider
)
seeds = as_async_generator(sync_seeds)
start = as_async_generator(sync_start)
else:
seeds = yield _maybe_sync_generator_to_async(spider.start)()
seeds = yield self._process_chain("process_start", seeds)
return seeds
start = yield _maybe_sync_generator_to_async(spider.start)()
start = yield self._process_chain("process_start", start)
return start
def _check_deprecated_start_requests_use(self, spider: Spider):
start_requests_cls = None

10
scrapy/templates/project/module/middlewares.py.tmpl
View File

@ -43,11 +43,11 @@ class ${ProjectName}SpiderMiddleware:
# Should return either None or an iterable of Request or item objects.
pass
async def process_start(self, seeds):
# Called with the seeds from the spider start() method or with
# the output of the maching method of an earlier spider middleware.
async for seed in seeds:
yield seed
async def process_start(self, start):
# Called with an async iterator over the spider start() method or the
# maching method of an earlier spider middleware.
async for item_or_request in start:
yield item_or_request
def spider_opened(self, spider):
spider.logger.info("Spider opened: %s" % spider.name)

8
tests/spiders.py
View File

@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider):
self.raise_exception()
class BrokenYieldSeedsSpider(FollowAllSpider):
class BrokenStartSpider(FollowAllSpider):
fail_before_yield = False
fail_yielding = False
@ -345,12 +345,12 @@ class BrokenYieldSeedsSpider(FollowAllSpider):
yield from super().parse(response)
class YieldSeedsItemSpider(FollowAllSpider):
class StartItemSpider(FollowAllSpider):
async def start(self):
yield {"name": "test item"}
class YieldSeedsGoodAndBadOutput(FollowAllSpider):
class StartGoodAndBadOutput(FollowAllSpider):
async def start(self):
yield {"a": "a"}
yield Request("data:,a")
@ -384,7 +384,7 @@ class SingleRequestSpider(MetaSpider):
return None
class DuplicateYieldSeedsSpider(MockServerSpider):
class DuplicateStartSpider(MockServerSpider):
dont_filter = True
name = "duplicatestartrequests"
distinct_urls = 2

22
tests/test_crawl.py
View File

@ -34,7 +34,7 @@ from tests.spiders import (
AsyncDefDeferredMaybeWrappedSpider,
AsyncDefDeferredWrappedSpider,
AsyncDefSpider,
BrokenYieldSeedsSpider,
BrokenStartSpider,
BytesReceivedCallbackSpider,
BytesReceivedErrbackSpider,
CrawlSpiderWithAsyncCallback,
@ -43,14 +43,14 @@ from tests.spiders import (
CrawlSpiderWithParseMethod,
CrawlSpiderWithProcessRequestCallbackKeywordArguments,
DelaySpider,
DuplicateYieldSeedsSpider,
DuplicateStartSpider,
FollowAllSpider,
HeadersReceivedCallbackSpider,
HeadersReceivedErrbackSpider,
SimpleSpider,
SingleRequestSpider,
YieldSeedsGoodAndBadOutput,
YieldSeedsItemSpider,
StartGoodAndBadOutput,
StartItemSpider,
)
@ -165,7 +165,7 @@ class TestCrawl(TestCase):
@defer.inlineCallbacks
def test_start_bug_before_yield(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenYieldSeedsSpider)
crawler = get_crawler(BrokenStartSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
assert len(log.records) == 1
@ -176,7 +176,7 @@ class TestCrawl(TestCase):
@defer.inlineCallbacks
def test_start_bug_yielding(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenYieldSeedsSpider)
crawler = get_crawler(BrokenStartSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
assert len(log.records) == 1
@ -187,7 +187,7 @@ class TestCrawl(TestCase):
@defer.inlineCallbacks
def test_start_items(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(YieldSeedsItemSpider)
crawler = get_crawler(StartItemSpider)
yield crawler.crawl(mockserver=self.mockserver)
assert len(log.records) == 0
@ -199,7 +199,7 @@ class TestCrawl(TestCase):
things fail when ItemAdapter is actually used on the corresponding
non-item object."""
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(YieldSeedsGoodAndBadOutput)
crawler = get_crawler(StartGoodAndBadOutput)
yield crawler.crawl(mockserver=self.mockserver)
assert len(log.records) == 0
@ -207,7 +207,7 @@ class TestCrawl(TestCase):
@defer.inlineCallbacks
def test_start_laziness(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = get_crawler(BrokenYieldSeedsSpider, settings)
crawler = get_crawler(BrokenStartSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(
99
@ -216,13 +216,13 @@ class TestCrawl(TestCase):
@defer.inlineCallbacks
def test_start_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = get_crawler(DuplicateYieldSeedsSpider, settings)
crawler = get_crawler(DuplicateStartSpider, settings)
yield crawler.crawl(
dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver
)
assert crawler.spider.visited == 6
crawler = get_crawler(DuplicateYieldSeedsSpider, settings)
crawler = get_crawler(DuplicateStartSpider, settings)
yield crawler.crawl(
dont_filter=False,
distinct_urls=3,

4
tests/test_engine.py
View File

@ -494,11 +494,11 @@ def test_request_scheduled_signal(caplog):
engine.downloader._slot_gc_loop.stop()
scheduler = TestScheduler()
async def seeds():
async def start():
return
yield
engine._seeds = seeds()
engine._start = start()
engine._slot = _Slot(False, Mock(), scheduler)
crawler.signals.connect(signal_handler, request_scheduled)
keep_request = Request("https://keep.example")

12
tests/test_engine_seeding.py
View File

@ -15,9 +15,9 @@ from .test_spider_start import twisted_sleep
class MainTestCase(TestCase):
@inlineCallbacks
def test_scheduler_priority_over_seeds_simple(self):
"""Scrapy reads seeds into the scheduler while the scheduler is empty,
but otherwise prioritizes requests already in the scheduler.
def test_scheduler_priority_over_start_simple(self):
"""Scrapy reads start() into the scheduler while the scheduler is
empty, but otherwise prioritizes requests already in the scheduler.
This test shows how, given a scheduler pre-filled with a request, that
request is sent before sending the first seed request.
@ -61,13 +61,13 @@ class MainTestCase(TestCase):
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
@inlineCallbacks
def test_scheduler_priority_over_seeds_complex(self):
"""Although Scrapy reads seeds into the scheduler while the scheduler
def test_scheduler_priority_over_start_complex(self):
"""Although Scrapy reads start() into the scheduler while the scheduler
is empty and otherwise prioritizes requests already in the scheduler,
this is done in a non-blocking way.
That is, if the scheduler reports having requests but yields none,
requests from seeds will be scheduled.
requests from start() will be scheduled.
"""
class TestScheduler(BaseScheduler):

2
tests/test_spider.py
View File

@ -460,7 +460,7 @@ class TestCrawlSpider(TestSpider):
(
"scrapy.core.engine",
"ERROR",
"Error while reading seeds",
"Error while reading start items and requests",
),
)

18
tests/test_spidermiddleware.py
View File

@ -320,19 +320,19 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
)
class ProcessYieldSeedsSimpleMiddleware:
class ProcessStartSimpleMiddleware:
def process_test_start(self, test_start, spider):
yield from test_start
class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware):
class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
"""process_start tests for simple start"""
ITEM_TYPE = (Request, dict)
MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware
MW_SIMPLE = ProcessStartSimpleMiddleware
@inlineCallbacks
def _get_processed_seeds(self, *mw_classes):
def _get_processed_start(self, *mw_classes):
class TestSpider(Spider):
name = "test"
@ -353,11 +353,11 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware):
@inlineCallbacks
def test_simple(self):
"""Simple mw"""
seeds = yield self._get_processed_seeds(self.MW_SIMPLE)
assert isasyncgen(seeds)
seed_list = yield deferred_from_coro(collect_asyncgen(seeds))
assert len(seed_list) == self.RESULT_COUNT
assert isinstance(seed_list[0], self.ITEM_TYPE)
start = yield self._get_processed_start(self.MW_SIMPLE)
assert isasyncgen(start)
start_list = yield deferred_from_coro(collect_asyncgen(start))
assert len(start_list) == self.RESULT_COUNT
assert isinstance(start_list[0], self.ITEM_TYPE)
class UniversalMiddlewareNoSync:

44
tests/test_spidermiddleware_process_start.py
View File

@ -17,29 +17,29 @@ ITEM_D = {"id": "d"}
class AsyncioSleepSpiderMiddleware:
async def process_start(self, seeds):
async def process_start(self, start):
await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)
async for seed in seeds:
yield seed
async for item_or_request in start:
yield item_or_request
class NoOpSpiderMiddleware:
async def process_start(self, seeds):
async for seed in seeds:
yield seed
async def process_start(self, start):
async for item_or_request in start:
yield item_or_request
class TwistedSleepSpiderMiddleware:
async def process_start(self, seeds):
async def process_start(self, start):
await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS))
async for seed in seeds:
yield seed
async for item_or_request in start:
yield item_or_request
class UniversalSpiderMiddleware:
async def process_start(self, seeds):
async for seed in seeds:
yield seed
async def process_start(self, start):
async for item_or_request in start:
yield item_or_request
def process_start_requests(self, start_requests, spider):
raise NotImplementedError
@ -73,30 +73,30 @@ class DeprecatedWrapSpider(Spider):
class ModernWrapSpiderMiddleware:
async def process_start(self, seeds):
async def process_start(self, start):
yield ITEM_A
async for seed in seeds:
yield seed
async for item_or_request in start:
yield item_or_request
yield ITEM_C
class UniversalWrapSpiderMiddleware:
async def process_start(self, seeds):
async def process_start(self, start):
yield ITEM_A
async for seed in seeds:
yield seed
async for item_or_request in start:
yield item_or_request
yield ITEM_C
def process_start_requests(self, seeds, spider):
def process_start_requests(self, start, spider):
yield ITEM_A
yield from seeds
yield from start
yield ITEM_C
class DeprecatedWrapSpiderMiddleware:
def process_start_requests(self, seeds, spider):
def process_start_requests(self, start, spider):
yield ITEM_A
yield from seeds
yield from start
yield ITEM_C