test_crawl.py: remove tests covered elsewhere

This commit is contained in:
Adrián Chaves 2025-03-15 02:38:28 +01:00
parent 5c6872a883
commit dbdfb1854d
2 changed files with 0 additions and 58 deletions

View File

@ -319,32 +319,6 @@ class ErrorSpider(FollowAllSpider):
self.raise_exception()
class BrokenYieldSeedsSpider(FollowAllSpider):
fail_before_yield = False
fail_yielding = False
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
self.seedsseen = []
async def yield_seeds(self):
if self.fail_before_yield:
1 / 0
for s in range(100):
qargs = {"total": 10, "seed": s}
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
yield Request(url, meta={"seed": s})
if self.fail_yielding:
2 / 0
assert self.seedsseen, "All seeds consumed before any download happened"
def parse(self, response):
self.seedsseen.append(response.meta.get("seed"))
yield from super().parse(response)
class YieldSeedsItemSpider(FollowAllSpider):
async def yield_seeds(self):
yield {"name": "test item"}

View File

@ -34,7 +34,6 @@ from tests.spiders import (
AsyncDefDeferredMaybeWrappedSpider,
AsyncDefDeferredWrappedSpider,
AsyncDefSpider,
BrokenYieldSeedsSpider,
BytesReceivedCallbackSpider,
BytesReceivedErrbackSpider,
CrawlSpiderWithAsyncCallback,
@ -162,28 +161,6 @@ class TestCrawl(TestCase):
)
self._assert_retried(log)
@defer.inlineCallbacks
def test_yield_seeds_bug_before_yield(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenYieldSeedsSpider)
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
assert len(log.records) == 1
record = log.records[0]
assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError
@defer.inlineCallbacks
def test_yield_seeds_bug_yielding(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
crawler = get_crawler(BrokenYieldSeedsSpider)
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
assert len(log.records) == 1
record = log.records[0]
assert record.exc_info is not None
assert record.exc_info[0] is ZeroDivisionError
@defer.inlineCallbacks
def test_yield_seeds_items(self):
with LogCapture("scrapy", level=logging.ERROR) as log:
@ -205,15 +182,6 @@ class TestCrawl(TestCase):
assert len(log.records) == 1
@defer.inlineCallbacks
def test_yield_seeds_laziness(self):
settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"}
crawler = get_crawler(BrokenYieldSeedsSpider, settings)
yield crawler.crawl(mockserver=self.mockserver)
assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(
99
), crawler.spider.seedsseen
@defer.inlineCallbacks
def test_yield_seeds_dupes(self):
settings = {"CONCURRENT_REQUESTS": 1}