diff --git a/docs/news.rst b/docs/news.rst index a82d05d67..efe2cd819 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,13 +11,16 @@ Scrapy VERSION (unreleased) Highlights: - Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.yield_seeds` - (async) + (async) and changed how it is iterated by default. Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, - ``process_start_requests()`` has been replaced by ``process_seeds()``. +- By default, the iteration of start requests and items no longer stops once + there are requests in the scheduler. + + You can restore the previous behavior by setting :setting:`SEEDING_POLICY` + to :py:enum:mem:`~scrapy.SeedingPolicy.lazy`. - In ``scrapy.core.engine.ExecutionEngine``: @@ -33,6 +36,9 @@ Backward-incompatible changes - The ``slot`` :ref:`telnet variable ` has been removed. +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_seeds()``. + Deprecations ~~~~~~~~~~~~ @@ -40,16 +46,14 @@ Deprecations use :meth:`~scrapy.Spider.yield_seeds` instead, or both to maintain support for lower Scrapy versions. - (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, - :issue:`6729`) + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) - The ``process_start_requests()`` method of :ref:`spider middlewares ` is deprecated, use :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` instead, or both to maintain support for lower Scrapy versions. - (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, - :issue:`6729`) + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) New features ~~~~~~~~~~~~ @@ -65,8 +69,8 @@ New features (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`) -- The new :setting:`SEEDING_POLICY` setting allows customizing how spider - start requests and items are consumed. +- The new :setting:`SEEDING_POLICY` setting allows customizing how start + requests and items are iterated. You can also override the active seeding policy from :meth:`Spider.yield_seeds ` and from @@ -87,7 +91,7 @@ Bug fixes equivalent) no longer delays the next iteration of starting requests and items by up to 5 seconds. - (:issue:`6715`, :issue:`6729`) + (:issue:`6729`) .. _release-2.12.0: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d773757d0..7aa340d8f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1740,7 +1740,7 @@ SEEDING_POLICY .. versionadded:: VERSION -Default: :py:enum:mem:`SeedingPolicy.lazy ` +Default: :py:enum:mem:`SeedingPolicy.greedy ` Determines the way :meth:`Spider.yield_seeds ` is iterated. diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 3047ae396..c50c963ef 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,6 +9,7 @@ from __future__ import annotations from threading import Thread from typing import TYPE_CHECKING, Any +from scrapy import SeedingPolicy from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -24,9 +25,10 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = False default_settings = { + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "SEEDING_POLICY": SeedingPolicy.lazy, } def syntax(self) -> str: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a08becfee..886154bfa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,6 +17,8 @@ import sys from importlib import import_module from pathlib import Path +from scrapy import SeedingPolicy + ADDONS = {} AJAXCRAWL_ENABLED = False @@ -308,7 +310,7 @@ SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 -SEEDING_POLICY = "lazy" +SEEDING_POLICY = SeedingPolicy.greedy SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 61662a702..442f04086 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -207,7 +207,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_yield_seeds_laziness(self): - settings = {"CONCURRENT_REQUESTS": 1} + settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"} crawler = get_crawler(BrokenYieldSeedsSpider, settings) yield crawler.crawl(mockserver=self.mockserver) assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index 9fbe4c6e7..c538fae44 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -31,6 +31,45 @@ class MainTestCase(TestCase): # eventually remove the heartbeat altogether. timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + @deferred_f_from_coro_f + async def test_greedy(self): + class TestScheduler(BaseScheduler): + def __init__(self, *args, **kwargs): + self.requests = deque((Request("data:,b"),)) + + def enqueue_request(self, request: Request) -> bool: + self.requests.append(request) + return True + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def next_request(self) -> Request | None: + try: + return self.requests.pop() + except IndexError: + return None + + class TestSpider(Spider): + name = "test" + start_urls = ["data:,a"] + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": TestScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + @deferred_f_from_coro_f async def test_lazy(self): class TestScheduler(BaseScheduler): @@ -62,7 +101,7 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - settings = {"SCHEDULER": TestScheduler} + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) await maybe_deferred_to_future(crawler.crawl()) @@ -115,7 +154,7 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - settings = {"SCHEDULER": TestScheduler} + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) await maybe_deferred_to_future(crawler.crawl()) @@ -140,50 +179,12 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - crawler = get_crawler(TestSpider) - crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) - assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b", "data:,c"] - assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - - @deferred_f_from_coro_f - async def test_greedy(self): - class TestScheduler(BaseScheduler): - def __init__(self, *args, **kwargs): - self.requests = deque((Request("data:,b"),)) - - def enqueue_request(self, request: Request) -> bool: - self.requests.append(request) - return True - - def has_pending_requests(self) -> bool: - return bool(self.requests) - - def next_request(self) -> Request | None: - try: - return self.requests.pop() - except IndexError: - return None - - class TestSpider(Spider): - name = "test" - start_urls = ["data:,a"] - - def parse(self, response): - pass - - actual_urls = [] - - def track_url(request, spider): - actual_urls.append(request.url) - - settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "greedy"} + settings = {"SEEDING_POLICY": "lazy"} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) await maybe_deferred_to_future(crawler.crawl()) assert crawler.stats.get_value("finish_reason") == "finished" - expected_urls = ["data:,a", "data:,b"] + expected_urls = ["data:,a", "data:,b", "data:,c"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" @deferred_f_from_coro_f @@ -276,7 +277,7 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - settings = {"SCHEDULER": TestScheduler} + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_item, signals.item_scraped) crawler.signals.connect(track_url, signals.request_reached_downloader)