From 01ed605d02013b1d7955369562b2443d2a561599 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 Mar 2019 16:54:14 +0000 Subject: [PATCH] PEP8 changes to test_spider.py --- tests/test_spider.py | 60 +++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 5e20e0d99..c9af7a2d7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -105,11 +105,11 @@ class SpiderTest(unittest.TestCase): def test_logger(self): spider = self.spider_class('example.com') - with LogCapture() as l: + with LogCapture() as lc: spider.logger.info('test log msg') - l.check(('example.com', 'INFO', 'test log msg')) + lc.check(('example.com', 'INFO', 'test log msg')) - record = l.records[0] + record = lc.records[0] self.assertIn('spider', record.__dict__) self.assertIs(record.spider, spider) @@ -190,12 +190,11 @@ class CrawlSpiderTest(SpiderTest): def test_process_links(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="dummy_process_links"), ) @@ -208,24 +207,24 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) def test_process_links_filter(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): import re - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="filter_process_links"), ) _test_regex = re.compile('nofollow') + def filter_process_links(self, links): return [link for link in links if not self._test_regex.search(link.url)] @@ -235,17 +234,16 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html']) def test_process_links_generator(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="dummy_process_links"), ) @@ -259,9 +257,9 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) def test_process_request(self): @@ -271,8 +269,8 @@ class CrawlSpiderTest(SpiderTest): return request.replace(url=request.url.replace('.org', '.com')) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request=process_request_change_domain), ) @@ -295,8 +293,8 @@ class CrawlSpiderTest(SpiderTest): return request class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request=process_request_meta_response_class), ) @@ -317,8 +315,8 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request='process_request_upper'), ) @@ -340,8 +338,8 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request='process_request_meta_response_class'), )