RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplify running spiders in CrawlSpiderTestCase.

This commit is contained in:
Andrey Rakhmatullin 2020-08-28 18:31:09 +05:00
parent 7e6476ff0a
commit 8123c42737
1 changed files with 21 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
tests/test_crawl.py
View File

@ -335,6 +335,19 @@ class CrawlSpiderTestCase(TestCase):
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def _run_spider(self, spider_cls):
items = []
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(spider_cls)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
return log, items, crawler.stats
@defer.inlineCallbacks
def test_crawlspider_with_parse(self):
self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
@ -379,15 +392,7 @@ class CrawlSpiderTestCase(TestCase):
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse_items_list(self):
items = []
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
log, items, _ = yield self._run_spider(AsyncDefAsyncioReturnSpider)
self.assertIn("Got response 200", str(log))
self.assertIn({'id': 1}, items)
self.assertIn({'id': 2}, items)
@ -410,27 +415,17 @@ class CrawlSpiderTestCase(TestCase):
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncgen_parse(self):
crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
log, _, stats = yield self._run_spider(AsyncDefAsyncioGenSpider)
self.assertIn("Got response 200", str(log))
itemcount = crawler.stats.get_value('item_scraped_count')
itemcount = stats.get_value('item_scraped_count')
self.assertEqual(itemcount, 1)
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_loop(self):
items = []
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
log, items, stats = yield self._run_spider(AsyncDefAsyncioGenLoopSpider)
self.assertIn("Got response 200", str(log))
itemcount = crawler.stats.get_value('item_scraped_count')
itemcount = stats.get_value('item_scraped_count')
self.assertEqual(itemcount, 10)
for i in range(10):
self.assertIn({'foo': i}, items)
@ -438,15 +433,8 @@ class CrawlSpiderTestCase(TestCase):
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncgen_parse_complex(self):
items = []
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
yield crawler.crawl(mockserver=self.mockserver)
itemcount = crawler.stats.get_value('item_scraped_count')
_, items, stats = yield self._run_spider(AsyncDefAsyncioGenComplexSpider)
itemcount = stats.get_value('item_scraped_count')
self.assertEqual(itemcount, 156)
# some random items
for i in [1, 4, 21, 22, 207, 311]:
@ -457,9 +445,7 @@ class CrawlSpiderTestCase(TestCase):
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse_reqs_list(self):
crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
log, *_ = yield self._run_spider(AsyncDefAsyncioReqsReturnSpider)
for req_id in range(3):
self.assertIn("Got response 200, req_id %d" % req_id, str(log))