RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Flake8: Remove E128 (tests, part 2)

This commit is contained in:
Eugenio Lacuesta 2020-05-06 19:18:29 -03:00
parent 9c6d13075a
commit bf56517abf
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
10 changed files with 94 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pytest.ini
View File

@ -200,19 +200,19 @@ flake8-ignore =
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_item.py E128 F841
tests/test_http_request.py E402 E501
tests/test_http_response.py E501
tests/test_item.py E501 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128
tests/test_loader.py E501 E741 E128 E117
tests/test_logformatter.py E128 E501
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128
tests/test_linkextractors.py E501
tests/test_loader.py E501 E741 E117
tests/test_logformatter.py E501
tests/test_mail.py E501
tests/test_middleware.py E501
tests/test_pipeline_crawl.py E501
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E128
tests/test_pipeline_media.py E501 E741
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
@ -221,8 +221,8 @@ flake8-ignore =
tests/test_selector.py E501
tests/test_spider.py E501
tests/test_spidermiddleware.py E501
tests/test_spidermiddleware_httperror.py E128 E501 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_httperror.py E501 E121
tests/test_spidermiddleware_offsite.py E501 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
tests/test_squeues.py E501 E741

22
tests/test_item.py
View File

@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase):
item = D(save='X', load='Y')
self.assertEqual(item['save'], 'X')
self.assertEqual(item['load'], 'Y')
self.assertEqual(D.fields, {'load': {'default': 'A'},
'save': {'default': 'A'}})
self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})
# D class inverted
class E(C, B):
@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase):
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
self.assertEqual(E.fields, {'load': {'default': 'C'},
'save': {'default': 'C'}})
self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})
def test_metaclass_multiple_inheritance_diamond(self):
class A(Item):
@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase):
self.assertEqual(D(save='X')['save'], 'X')
self.assertEqual(D(load='X')['load'], 'X')
self.assertEqual(D.fields, {'save': {'default': 'C'},
'load': {'default': 'D'}, 'update': {'default': 'D'}})
self.assertEqual(
D.fields,
{'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})
# D class inverted
class E(C, B):
@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase):
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
self.assertEqual(E.fields, {'save': {'default': 'C'},
'load': {'default': 'E'}, 'update': {'default': 'C'}})
self.assertEqual(
E.fields,
{'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})
def test_metaclass_multiple_inheritance_without_metaclass(self):
class A(Item):
@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase):
self.assertRaises(KeyError, D, not_allowed='value')
self.assertEqual(D(save='X')['save'], 'X')
self.assertEqual(D.fields, {'save': {'default': 'A'},
'load': {'default': 'A'}})
self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
# D class inverted
class E(C, B):
@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase):
self.assertRaises(KeyError, E, not_allowed='value')
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E.fields, {'save': {'default': 'A'},
'load': {'default': 'A'}})
self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
def test_to_dict(self):
class TestItem(Item):

6
tests/test_linkextractors.py
View File

@ -171,9 +171,9 @@ class Base:
self.assertEqual(lx.matches(url1), False)
self.assertEqual(lx.matches(url2), True)
lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
allow_domains=('blah1.com',),
deny_domains=('blah2.com',))
lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],
allow_domains=['blah1.com'],
deny_domains=['blah2.com'])
self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)

21
tests/test_logformatter.py
View File

@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> (referer: None)")
self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
def test_crawled_without_referer(self):
req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
res = Response("http://www.example.com", flags=['cached'])
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
def test_dropped(self):
item = {}
@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: None) []")
def test_crawled_without_referer(self):
@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
class SkipMessagesLogFormatter(LogFormatter):

2
tests/test_mail.py
View File

@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase):
mailsender = MailSender(debug=True)
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
attachs=attachs, _callback=self._catch_mail_sent)
attachs=attachs, _callback=self._catch_mail_sent)
assert self.catched_msg
self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])

9
tests/test_middleware.py
View File

@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())
self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['open_spider']],
[M1, M2])
self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['close_spider']],
[M2, M1])
self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['process']],
[M1, M3])
def test_enabled(self):

12
tests/test_pipeline_crawl.py
View File

@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media(self):
crawler = self._create_crawler(MediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key)
self._assert_files_downloaded(self.items, str(log))
@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_wrong_urls(self):
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key)
self._assert_files_download_failure(crawler, self.items, 404, str(log))
@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_redirected_default_failure(self):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
mockserver=self.mockserver)
@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
mockserver=self.mockserver)

12
tests/test_pipeline_media.py
View File

@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, rsp)])
self.assertEqual(self.pipe._mockcalled,
['get_media_requests', 'media_to_download',
'media_downloaded', 'request_callback', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
@inlineCallbacks
def test_result_failure(self):
@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(False, fail)])
self.assertEqual(self.pipe._mockcalled,
['get_media_requests', 'media_to_download',
'media_failed', 'request_errback', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
@inlineCallbacks
def test_mix_of_success_and_failure(self):

55
tests/test_spidermiddleware_httperror.py
View File

@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase):
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEqual([],
self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEqual(None,
self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
self.assertEqual(
[],
self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
self.assertEqual(None,
self.mw.process_spider_input(res, self.spider))
self.assertIsNone(self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
class TestHttpErrorMiddlewareSettings(TestCase):
@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase):
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertEqual(None,
self.mw.process_spider_input(self.res402, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase):
res402 = self.res402.copy()
res402.request = request
self.assertEqual(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
class TestHttpErrorMiddlewareHandleAll(TestCase):
@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402 = self.res402.copy()
res402.request = request
self.assertEqual(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):

32
tests/test_spidermiddleware_offsite.py
View File

@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase):
def test_process_spider_output(self):
res = Response('http://scrapytest.org')
onsite_reqs = [Request('http://scrapytest.org/1'),
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
Request('http://scrapy.test.org/'),
Request('http://scrapy.test.org:8000/')]
offsite_reqs = [Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org'),
Request('http://test.org/'),
Request('http://notscrapy.test.org/')]
onsite_reqs = [
Request('http://scrapytest.org/1'),
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
Request('http://scrapy.test.org/'),
Request('http://scrapy.test.org:8000/'),
]
offsite_reqs = [
Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org'),
Request('http://test.org/'),
Request('http://notscrapy.test.org/'),
]
reqs = onsite_reqs + offsite_reqs
out = list(self.mw.process_spider_output(res, reqs, self.spider))