RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Flake8: Remove E128 (tests, part 3)

This commit is contained in:
Eugenio Lacuesta 2020-05-06 19:33:22 -03:00
parent bf56517abf
commit 881b4f417f
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
9 changed files with 87 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pytest.ini
View File

@ -227,23 +227,23 @@ flake8-ignore =
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E128
tests/test_utils_conf.py E501
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
tests/test_utils_http.py E501 E128
tests/test_utils_iterators.py E501 E128 E129
tests/test_utils_http.py E501
tests/test_utils_iterators.py E501 E129
tests/test_utils_log.py E741
tests/test_utils_python.py E501
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128
tests/test_utils_reqser.py E501
tests/test_utils_request.py E501
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841
tests/test_utils_sitemap.py E128 E501
tests/test_utils_sitemap.py E501
tests/test_utils_url.py E501 E501
tests/test_webclient.py E501 E128 E402
tests/test_webclient.py E501 E402
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_settings/__init__.py E501
tests/test_spiderloader/__init__.py E501
tests/test_utils_misc/__init__.py E501

18
tests/test_settings/__init__.py
View File

@ -314,13 +314,17 @@ class BaseSettingsTest(unittest.TestCase):
'TEST_BASE': BaseSettings({1: 1, 2: 2}, 'project'),
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
self.assertDictEqual(s.copy_to_dict(),
{'HASNOBASE': {3: 3000},
'TEST': {1: 10, 3: 30},
'TEST_BASE': {1: 1, 2: 2},
'TEST_BOOLEAN': False,
'TEST_LIST': [1, 2],
'TEST_STRING': 'a string'})
self.assertDictEqual(
s.copy_to_dict(),
{
'HASNOBASE': {3: 3000},
'TEST': {1: 10, 3: 30},
'TEST_BASE': {1: 1, 2: 2},
'TEST_LIST': [1, 2],
'TEST_BOOLEAN': False,
'TEST_STRING': 'a string',
}
)
def test_freeze(self):
self.settings.freeze()

21
tests/test_spiderloader/__init__.py
View File

@ -40,7 +40,8 @@ class SpiderLoaderTest(unittest.TestCase):
verifyObject(ISpiderLoader, self.spider_loader)
def test_list(self):
self.assertEqual(set(self.spider_loader.list()),
self.assertEqual(
set(self.spider_loader.list()),
set(['spider1', 'spider2', 'spider3', 'spider4']))
def test_load(self):
@ -48,17 +49,23 @@ class SpiderLoaderTest(unittest.TestCase):
self.assertEqual(spider1.__name__, 'Spider1')
def test_find_by_request(self):
self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
['spider1'])
self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
['spider2'])
self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
self.assertEqual(
set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
set(['spider1', 'spider2']))
self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
[])
self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://spider3.com')),
[])
self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
self.assertEqual(
self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
['spider3'])
def test_load_spider_module(self):

3
tests/test_utils_conf.py
View File

@ -93,7 +93,8 @@ class BuildComponentListTest(unittest.TestCase):
class UtilsConfTestCase(unittest.TestCase):
def test_arglist_to_dict(self):
self.assertEqual(arglist_to_dict(['arg1=val1', 'arg2=val2']),
self.assertEqual(
arglist_to_dict(['arg1=val1', 'arg2=val2']),
{'arg1': 'val1', 'arg2': 'val2'})

23
tests/test_utils_iterators.py
View File

@ -47,8 +47,7 @@ class XmliterTestCase(unittest.TestCase):
</root>
"""
response = XmlResponse(url="http://example.com", body=body)
nodenames = [e.xpath('name()').getall()
for e in self.xmliter(response, 'matchme...')]
nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')]
self.assertEqual(nodenames, [['matchme...']])
def test_xmliter_unicode(self):
@ -359,15 +358,23 @@ class UtilsCsvTestCase(unittest.TestCase):
response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'latin1', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'}])
self.assertEqual(
list(csv),
[
{u'id': u'1', u'name': u'latin1', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\xf1\xe1\xe9\xf3'},
]
)
response = TextResponse(url="http://example.com/", body=body2, encoding='cp852')
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'cp852', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'}])
self.assertEqual(
list(csv),
[
{u'id': u'1', u'name': u'cp852', u'value': u'test'},
{u'id': u'2', u'name': u'something', u'value': u'\u255a\u2569\u2569\u2569\u2550\u2550\u2557'},
]
)
class TestHelper(unittest.TestCase):

3
tests/test_utils_reqser.py
View File

@ -15,7 +15,8 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r)
def test_all_attributes(self):
r = Request("http://www.example.com",
r = Request(
url="http://www.example.com",
callback=self.spider.parse_item,
errback=self.spider.handle_error,
method="POST",

5
tests/test_utils_request.py
View File

@ -36,8 +36,9 @@ class UtilsRequestTest(unittest.TestCase):
self.assertEqual(request_fingerprint(r1),
request_fingerprint(r1, include_headers=['Accept-Language']))
self.assertNotEqual(request_fingerprint(r1),
request_fingerprint(r2, include_headers=['Accept-Language']))
self.assertNotEqual(
request_fingerprint(r1),
request_fingerprint(r2, include_headers=['Accept-Language']))
self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))

9
tests/test_utils_sitemap.py
View File

@ -22,8 +22,13 @@ class SitemapTest(unittest.TestCase):
</url>
</urlset>""")
assert s.type == 'urlset'
self.assertEqual(list(s),
[{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'}, {'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'}])
self.assertEqual(
list(s),
[
{'priority': '1', 'loc': 'http://www.example.com/', 'lastmod': '2009-08-16', 'changefreq': 'daily'},
{'priority': '0.8', 'loc': 'http://www.example.com/Special-Offers.html', 'lastmod': '2009-08-16', 'changefreq': 'weekly'},
]
)
def test_sitemap_index(self):
s = Sitemap(b"""<?xml version="1.0" encoding="UTF-8"?>

40
tests/test_webclient.py
View File

@ -47,8 +47,9 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
return f
from twisted.web.client import _makeGetterFactory
return _makeGetterFactory(to_bytes(url), _clientfactory,
contextFactory=contextFactory, *args, **kwargs).deferred
return _makeGetterFactory(
to_bytes(url), _clientfactory, contextFactory=contextFactory, *args, **kwargs
).deferred
class ParseUrlTestCase(unittest.TestCase):
@ -105,7 +106,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
'Content-Length': '12981',
'Useful': 'value'}))
self._test(factory,
self._test(
factory,
b"GET /bar HTTP/1.0\r\n"
b"Content-Length: 9\r\n"
b"Useful: value\r\n"
@ -118,7 +120,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
# test minimal sent headers
factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
self._test(factory,
self._test(
factory,
b"GET /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"\r\n")
@ -130,7 +133,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
body='name=value',
headers={'Content-Type': 'application/x-www-form-urlencoded'}))
self._test(factory,
self._test(
factory,
b"POST /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"Connection: close\r\n"
@ -145,7 +149,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
url='http://foo/bar'
))
self._test(factory,
self._test(
factory,
b"POST /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"Content-Length: 0\r\n"
@ -160,7 +165,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
},
))
self._test(factory,
self._test(
factory,
b"GET /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"X-Meta-Multivalued: value1\r\n"
@ -177,7 +183,8 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
}),
))
self._test(factory,
self._test(
factory,
b"GET /bar HTTP/1.0\r\n"
b"Host: foo\r\n"
b"X-Meta-Multivalued: value1\r\n"
@ -206,8 +213,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
protocol.dataReceived(b"Hello: World\n")
protocol.dataReceived(b"Foo: Bar\n")
protocol.dataReceived(b"\n")
self.assertEqual(protocol.headers,
Headers({'Hello': ['World'], 'Foo': ['Bar']}))
self.assertEqual(protocol.headers, Headers({'Hello': ['World'], 'Foo': ['Bar']}))
class EncodingResource(resource.Resource):
@ -340,10 +346,11 @@ class WebClientTestCase(unittest.TestCase):
return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
def _cbRedirect(self, pageData):
self.assertEqual(pageData,
b'\n<html>\n <head>\n <meta http-equiv="refresh" content="0;URL=/file">\n'
b' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
b'<a href="/file">click here</a>\n </body>\n</html>\n')
self.assertEqual(
pageData,
b'\n<html>\n <head>\n <meta http-equiv="refresh" content="0;URL=/file">\n'
b' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
b'<a href="/file">click here</a>\n </body>\n</html>\n')
def test_encoding(self):
""" Test that non-standart body encoding matches
@ -403,8 +410,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
s = "0123456789" * 10
settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': self.custom_ciphers})
client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
return getPage(self.getURL("payload"), body=s,
contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))
return getPage(
self.getURL("payload"), body=s, contextFactory=client_context_factory
).addCallback(self.assertEqual, to_bytes(s))
def testPayloadDefaultCiphers(self):
s = "0123456789" * 10