diff --git a/pytest.ini b/pytest.ini
index 0b2766031..7c3c9cc5b 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -227,23 +227,23 @@ flake8-ignore =
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
- tests/test_utils_conf.py E501 E128
+ tests/test_utils_conf.py E501
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
- tests/test_utils_http.py E501 E128
- tests/test_utils_iterators.py E501 E128 E129
+ tests/test_utils_http.py E501
+ tests/test_utils_iterators.py E501 E129
tests/test_utils_log.py E741
tests/test_utils_python.py E501
- tests/test_utils_reqser.py E501 E128
- tests/test_utils_request.py E501 E128
+ tests/test_utils_reqser.py E501
+ tests/test_utils_request.py E501
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841
- tests/test_utils_sitemap.py E128 E501
+ tests/test_utils_sitemap.py E501
tests/test_utils_url.py E501 E501
- tests/test_webclient.py E501 E128 E402
+ tests/test_webclient.py E501 E402
tests/test_cmdline/__init__.py E501
- tests/test_settings/__init__.py E501 E128
- tests/test_spiderloader/__init__.py E128 E501
+ tests/test_settings/__init__.py E501
+ tests/test_spiderloader/__init__.py E501
tests/test_utils_misc/__init__.py E501
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index fda44653a..2da6aa4b5 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase):
'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
- self.assertDictEqual(s.copy_to_dict(),
- {'HASNOBASE': {3: 3000},
- 'TEST': {1: 10, 3: 30},
- 'TEST_BASE': {1: 1, 2: 2},
- 'TEST_BOOLEAN': False,
- 'TEST_LIST': [1, 2],
- 'TEST_STRING': 'a string'})
+ self.assertDictEqual(
+ s.copy_to_dict(),
+ {
+ 'HASNOBASE': {3: 3000},
+ 'TEST': {1: 10, 3: 30},
+ 'TEST_BASE': {1: 1, 2: 2},
+ 'TEST_LIST': [1, 2],
+ 'TEST_BOOLEAN': False,
+ 'TEST_STRING': 'a string',
+ }
+ )
def test_freeze(self):
self.settings.freeze()
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d8be6e277..b20e1b3f2 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase):
verifyObject(ISpiderLoader, self.spider_loader)
def test_list(self):
- self.assertEqual(set(self.spider_loader.list()),
+ self.assertEqual(
+ set(self.spider_loader.list()),
set(['spider1', 'spider2', 'spider3', 'spider4']))
def test_load(self):
@@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase):
self.assertEqual(spider1.__name__, 'Spider1')
def test_find_by_request(self):
- self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
+ self.assertEqual(
+ self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
['spider1'])
- self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
+ self.assertEqual(
+ self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
['spider2'])
- self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
+ self.assertEqual(
+ set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
set(['spider1', 'spider2']))
- self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
+ self.assertEqual(
+ self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
[])
- self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
+ self.assertEqual(
+ self.spider_loader.find_by_request(Request('http://spider3.com')),
[])
- self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
+ self.assertEqual(
+ self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
['spider3'])
def test_load_spider_module(self):
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 332120021..e5d3ef582 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase):
class UtilsConfTestCase(unittest.TestCase):
def test_arglist_to_dict(self):
- self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
+ self.assertEqual(
+ arglist_to_dict(['arg1=val1', 'arg2=val2']),
{'arg1': 'val1', 'arg2': 'val2'})
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a85087619..c677dabe5 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -47,8 +47,7 @@ class XmliterTestCase(unittest.TestCase):
"""
response = XmlResponse(url="http://example.com", body=body)
- nodenames = [e.xpath('name()').getall()
- for e in self.xmliter(response, 'matchme...')]
+ nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
self.assertEqual(nodenames, [['matchme...']])
def test_xmliter_unicode(self):
@@ -359,15 +358,23 @@ class UtilsCsvTestCase(unittest.TestCase):
response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
csv = csviter(response)
- self.assertEqual([row for row in csv],
- [{u'id': u'1', u'name': u'latin1', u'value': u'test'},
- {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}])
+ self.assertEqual(
+ list(csv),
+ [
+ {u'id': u'1', u'name': u'latin1', u'value': u'test'},
+ {u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
+ ]
+ )
response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
csv = csviter(response)
- self.assertEqual([row for row in csv],
- [{u'id': u'1', u'name': u'cp852', u'value': u'test'},
- {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}])
+ self.assertEqual(
+ list(csv),
+ [
+ {u'id': u'1', u'name': u'cp852', u'value': u'test'},
+ {u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
+ ]
+ )
class TestHelper(unittest.TestCase):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 50b026d1c..450e4bdca 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r)
def test_all_attributes(self):
- r = Request("http://www.example.com",
+ r = Request(
+ url="http://www.example.com",
callback=self.spider.parse_item,
errback=self.spider.handle_error,
method="POST",
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 45f0f59e4..50efb63ca 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase):
self.assertEqual(request_fingerprint(r1),
request_fingerprint(r1, include_headers=['Accept-Language']))
- self.assertNotEqual(request_fingerprint(r1),
- request_fingerprint(r2, include_headers=['Accept-Language']))
+ self.assertNotEqual(
+ request_fingerprint(r1),
+ request_fingerprint(r2, include_headers=['Accept-Language']))
self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 08b215434..d77978ff1 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase):
""")
assert s.type == 'urlset'
- self.assertEqual(list(s),
- [{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
+ self.assertEqual(
+ list(s),
+ [
+ {'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
+ {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
+ ]
+ )
def test_sitemap_index(self):
s = Sitemap(b"""
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 307fadb5c..188e54602 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -47,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
return f
from twisted.web.client import _makeGetterFactory
- return _makeGetterFactory(to_bytes(url), _clientfactory,
- contextFactory=contextFactory, *args, **kwargs).deferred
+ return _makeGetterFactory(
+ to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs
+ ).deferred
class ParseUrlTestCase(unittest.TestCase):
@@ -105,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
'Content-Length': '12981',
'Useful': 'value'}))
- self._test(factory,
+ self._test(
+ factory,
b"GET /bar HTTP/1.0\r\n"
b"Content-Length: 9\r\n"
b"Useful: value\r\n"
@@ -118,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
# test minimal sent headers
factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
- self._test(factory,
+ self._test(
+ factory,
b"GET /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"\r\n")
@@ -130,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
body='name=value',
headers={'Content-Type': 'application/x-www-form-urlencoded'}))
- self._test(factory,
+ self._test(
+ factory,
b"POST /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"Connection: close\r\n"
@@ -145,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
url='http://foo/bar'
))
- self._test(factory,
+ self._test(
+ factory,
b"POST /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"Content-Length: 0\r\n"
@@ -160,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
},
))
- self._test(factory,
+ self._test(
+ factory,
b"GET /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"X-Meta-Multivalued: value1\r\n"
@@ -177,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
}),
))
- self._test(factory,
+ self._test(
+ factory,
b"GET /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"X-Meta-Multivalued: value1\r\n"
@@ -206,8 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
protocol.dataReceived(b"Hello: World\n")
protocol.dataReceived(b"Foo: Bar\n")
protocol.dataReceived(b"\n")
- self.assertEqual(protocol.headers,
- Headers({'Hello': ['World'], 'Foo': ['Bar']}))
+ self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']}))
class EncodingResource(resource.Resource):
@@ -340,10 +346,11 @@ class WebClientTestCase(unittest.TestCase):
return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
def _cbRedirect(self, pageData):
- self.assertEqual(pageData,
- b'\n\n \n \n'
- b' \n \n '
- b'click here\n \n\n')
+ self.assertEqual(
+ pageData,
+ b'\n\n \n \n'
+ b' \n \n '
+ b'click here\n \n\n')
def test_encoding(self):
""" Test that non-standart body encoding matches
@@ -403,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
s = "0123456789" * 10
settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers})
client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
- return getPage(self.getURL("payload"), body=s,
- contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))
+ return getPage(
+ self.getURL("payload"), body=s, contextFactory=client_context_factory
+ ).addCallback(self.assertEqual, to_bytes(s))
def testPayloadDefaultCiphers(self):
s = "0123456789" * 10