mirror of https://github.com/scrapy/scrapy.git
test_crawl.py: remove tests covered elsewhere
This commit is contained in:
parent
5c6872a883
commit
dbdfb1854d
|
|
@ -319,32 +319,6 @@ class ErrorSpider(FollowAllSpider):
|
|||
self.raise_exception()
|
||||
|
||||
|
||||
class BrokenYieldSeedsSpider(FollowAllSpider):
|
||||
fail_before_yield = False
|
||||
fail_yielding = False
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super().__init__(*a, **kw)
|
||||
self.seedsseen = []
|
||||
|
||||
async def yield_seeds(self):
|
||||
if self.fail_before_yield:
|
||||
1 / 0
|
||||
|
||||
for s in range(100):
|
||||
qargs = {"total": 10, "seed": s}
|
||||
url = self.mockserver.url(f"/follow?{urlencode(qargs, doseq=True)}")
|
||||
yield Request(url, meta={"seed": s})
|
||||
if self.fail_yielding:
|
||||
2 / 0
|
||||
|
||||
assert self.seedsseen, "All seeds consumed before any download happened"
|
||||
|
||||
def parse(self, response):
|
||||
self.seedsseen.append(response.meta.get("seed"))
|
||||
yield from super().parse(response)
|
||||
|
||||
|
||||
class YieldSeedsItemSpider(FollowAllSpider):
|
||||
async def yield_seeds(self):
|
||||
yield {"name": "test item"}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,6 @@ from tests.spiders import (
|
|||
AsyncDefDeferredMaybeWrappedSpider,
|
||||
AsyncDefDeferredWrappedSpider,
|
||||
AsyncDefSpider,
|
||||
BrokenYieldSeedsSpider,
|
||||
BytesReceivedCallbackSpider,
|
||||
BytesReceivedErrbackSpider,
|
||||
CrawlSpiderWithAsyncCallback,
|
||||
|
|
@ -162,28 +161,6 @@ class TestCrawl(TestCase):
|
|||
)
|
||||
self._assert_retried(log)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_yield_seeds_bug_before_yield(self):
|
||||
with LogCapture("scrapy", level=logging.ERROR) as log:
|
||||
crawler = get_crawler(BrokenYieldSeedsSpider)
|
||||
yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
|
||||
|
||||
assert len(log.records) == 1
|
||||
record = log.records[0]
|
||||
assert record.exc_info is not None
|
||||
assert record.exc_info[0] is ZeroDivisionError
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_yield_seeds_bug_yielding(self):
|
||||
with LogCapture("scrapy", level=logging.ERROR) as log:
|
||||
crawler = get_crawler(BrokenYieldSeedsSpider)
|
||||
yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
|
||||
|
||||
assert len(log.records) == 1
|
||||
record = log.records[0]
|
||||
assert record.exc_info is not None
|
||||
assert record.exc_info[0] is ZeroDivisionError
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_yield_seeds_items(self):
|
||||
with LogCapture("scrapy", level=logging.ERROR) as log:
|
||||
|
|
@ -205,15 +182,6 @@ class TestCrawl(TestCase):
|
|||
|
||||
assert len(log.records) == 1
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_yield_seeds_laziness(self):
|
||||
settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"}
|
||||
crawler = get_crawler(BrokenYieldSeedsSpider, settings)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(
|
||||
99
|
||||
), crawler.spider.seedsseen
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_yield_seeds_dupes(self):
|
||||
settings = {"CONCURRENT_REQUESTS": 1}
|
||||
|
|
|
|||
Loading…
Reference in New Issue