RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PEP8 changes to test_spider.py

This commit is contained in:
Eugenio Lacuesta 2019-03-15 16:54:14 +00:00
parent 83ec947fe7
commit 01ed605d02
1 changed files with 29 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
tests/test_spider.py
View File

@ -105,11 +105,11 @@ class SpiderTest(unittest.TestCase):
def test_logger(self):
spider = self.spider_class('example.com')
with LogCapture() as l:
with LogCapture() as lc:
spider.logger.info('test log msg')
l.check(('example.com', 'INFO', 'test log msg'))
lc.check(('example.com', 'INFO', 'test log msg'))
record = l.records[0]
record = lc.records[0]
self.assertIn('spider', record.__dict__)
self.assertIs(record.spider, spider)
@ -190,12 +190,11 @@ class CrawlSpiderTest(SpiderTest):
def test_process_links(self):
response = HtmlResponse("http://example.org/somepage/index.html",
body=self.test_body)
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
class _CrawlSpider(self.spider_class):
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_links="dummy_process_links"),
)
@ -208,24 +207,24 @@ class CrawlSpiderTest(SpiderTest):
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.org/somepage/item/12.html',
'http://example.org/about.html',
'http://example.org/nofollow.html'])
['http://example.org/somepage/item/12.html',
'http://example.org/about.html',
'http://example.org/nofollow.html'])
def test_process_links_filter(self):
response = HtmlResponse("http://example.org/somepage/index.html",
body=self.test_body)
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
class _CrawlSpider(self.spider_class):
import re
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_links="filter_process_links"),
)
_test_regex = re.compile('nofollow')
def filter_process_links(self, links):
return [link for link in links
if not self._test_regex.search(link.url)]
@ -235,17 +234,16 @@ class CrawlSpiderTest(SpiderTest):
self.assertEqual(len(output), 2)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.org/somepage/item/12.html',
'http://example.org/about.html'])
['http://example.org/somepage/item/12.html',
'http://example.org/about.html'])
def test_process_links_generator(self):
response = HtmlResponse("http://example.org/somepage/index.html",
body=self.test_body)
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
class _CrawlSpider(self.spider_class):
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_links="dummy_process_links"),
)
@ -259,9 +257,9 @@ class CrawlSpiderTest(SpiderTest):
self.assertEqual(len(output), 3)
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
self.assertEqual([r.url for r in output],
['http://example.org/somepage/item/12.html',
'http://example.org/about.html',
'http://example.org/nofollow.html'])
['http://example.org/somepage/item/12.html',
'http://example.org/about.html',
'http://example.org/nofollow.html'])
def test_process_request(self):
@ -271,8 +269,8 @@ class CrawlSpiderTest(SpiderTest):
return request.replace(url=request.url.replace('.org', '.com'))
class _CrawlSpider(self.spider_class):
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_request=process_request_change_domain),
)
@ -295,8 +293,8 @@ class CrawlSpiderTest(SpiderTest):
return request
class _CrawlSpider(self.spider_class):
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_request=process_request_meta_response_class),
)
@ -317,8 +315,8 @@ class CrawlSpiderTest(SpiderTest):
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
class _CrawlSpider(self.spider_class):
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_request='process_request_upper'),
)
@ -340,8 +338,8 @@ class CrawlSpiderTest(SpiderTest):
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
class _CrawlSpider(self.spider_class):
name="test"
allowed_domains=['example.org']
name = "test"
allowed_domains = ['example.org']
rules = (
Rule(LinkExtractor(), process_request='process_request_meta_response_class'),
)