diff --git a/tests/spiders.py b/tests/spiders.py index 5075a795b..be3580e5a 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -319,32 +319,6 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenYieldSeedsSpider(FollowAllSpider): - fail_before_yield = False - fail_yielding = False - - def __init__(self, *a, **kw): - super().__init__(*a, **kw) - self.seedsseen = [] - - async def yield_seeds(self): - if self.fail_before_yield: - 1 / 0 - - for s in range(100): - qargs = {"total": 10, "seed": s} - url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}") - yield Request(url, meta={"seed": s}) - if self.fail_yielding: - 2 / 0 - - assert self.seedsseen, "All seeds consumed before any download happened" - - def parse(self, response): - self.seedsseen.append(response.meta.get("seed")) - yield from super().parse(response) - - class YieldSeedsItemSpider(FollowAllSpider): async def yield_seeds(self): yield {"name": "test item"} diff --git a/tests/test_crawl.py b/tests/test_crawl.py index a2b66ac20..31c5618ad 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -34,7 +34,6 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenYieldSeedsSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -162,28 +161,6 @@ class TestCrawl(TestCase): ) self._assert_retried(log) - @defer.inlineCallbacks - def test_yield_seeds_bug_before_yield(self): - with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenYieldSeedsSpider) - yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) - - assert len(log.records) == 1 - record = log.records[0] - assert record.exc_info is not None - assert record.exc_info[0] is ZeroDivisionError - - @defer.inlineCallbacks - def test_yield_seeds_bug_yielding(self): - with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenYieldSeedsSpider) - yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) - - assert len(log.records) == 1 - record = log.records[0] - assert record.exc_info is not None - assert record.exc_info[0] is ZeroDivisionError - @defer.inlineCallbacks def test_yield_seeds_items(self): with LogCapture("scrapy", level=logging.ERROR) as log: @@ -205,15 +182,6 @@ class TestCrawl(TestCase): assert len(log.records) == 1 - @defer.inlineCallbacks - def test_yield_seeds_laziness(self): - settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"} - crawler = get_crawler(BrokenYieldSeedsSpider, settings) - yield crawler.crawl(mockserver=self.mockserver) - assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( - 99 - ), crawler.spider.seedsseen - @defer.inlineCallbacks def test_yield_seeds_dupes(self): settings = {"CONCURRENT_REQUESTS": 1}