RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[test] Override CrawlSpider.parse

This commit is contained in:
Eugenio Lacuesta 2019-12-26 14:38:11 -03:00
parent 5982e3477c
commit 8d4948f6ca
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 56 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
tests/spiders.py
View File

@ -186,13 +186,39 @@ class DuplicateStartRequestsSpider(MockServerSpider):
self.visited += 1
class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
name = 'crawl_spider_with_errback'
class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
"""
A CrawlSpider which overrides the 'parse' method
"""
name = 'crawl_spider_with_parse_method'
custom_settings = {
'RETRY_HTTP_CODES': [], # no need to retry
}
rules = (
Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),
Rule(LinkExtractor(), callback='parse', follow=True),
)
def start_requests(self):
test_body = b"""
<html>
<head><title>Page title<title></head>
<body>
<p><a href="/status?n=200">Item 200</a></p> <!-- callback -->
<p><a href="/status?n=201">Item 201</a></p> <!-- callback -->
</body>
</html>
"""
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
def parse(self, response):
self.logger.info('[parse] status %i', response.status)
class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
name = 'crawl_spider_with_errback'
rules = (
Rule(LinkExtractor(), callback='parse', errback='errback', follow=True),
)
def start_requests(self):
@ -211,8 +237,5 @@ class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
url = self.mockserver.url("/alpayload")
yield Request(url, method="POST", body=test_body)
def callback(self, response):
self.logger.info('[callback] status %i', response.status)
def errback(self, failure):
self.logger.info('[errback] status %i', failure.value.response.status)

31
tests/test_crawl.py
View File

@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
from tests.spiders import (BrokenStartRequestsSpider, CrawlSpiderWithErrback,
CrawlSpiderWithParseMethod, DelaySpider, SimpleSpider,
DuplicateStartRequestsSpider, FollowAllSpider, SingleRequestSpider)
class CrawlTestCase(TestCase):
@ -297,6 +298,27 @@ with multiples lines
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
class CrawlSpiderTestCase(TestCase):
def setUp(self):
self.mockserver = MockServer()
self.mockserver.__enter__()
self.runner = CrawlerRunner()
def tearDown(self):
self.mockserver.__exit__(None, None, None)
@defer.inlineCallbacks
def test_crawlspider_with_parse(self):
self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
with LogCapture() as log:
yield self.runner.join()
self.assertIn("[parse] status 200", str(log))
self.assertIn("[parse] status 201", str(log))
@defer.inlineCallbacks
def test_crawlspider_with_errback(self):
self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
@ -304,7 +326,8 @@ with multiples lines
with LogCapture() as log:
yield self.runner.join()
self.assertIn("[callback] status 200", str(log))
self.assertIn("[callback] status 201", str(log))
self.assertIn("[parse] status 200", str(log))
self.assertIn("[parse] status 201", str(log))
self.assertIn("[errback] status 404", str(log))
self.assertIn("[errback] status 500", str(log))
self.assertIn("[errback] status 501", str(log))