RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4365 from noviluni/fix_flake8_E701_E271 

Fix flake8 E131, E211, E251, E271, E701
This commit is contained in:
Mikhail Korobov 2020-02-23 15:52:52 +05:00 committed by GitHub
parent 9d983c1b99 6e8e117aee
commit 49480225d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 46 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
pytest.ini
View File

@ -128,7 +128,7 @@ flake8-ignore =
scrapy/utils/gz.py E501 W504
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/iterators.py E501
scrapy/utils/log.py E128 E501
scrapy/utils/markup.py F403
scrapy/utils/misc.py E501
@ -141,7 +141,7 @@ flake8-ignore =
scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
scrapy/utils/spider.py E271 E501
scrapy/utils/spider.py E501
scrapy/utils/ssl.py E501
scrapy/utils/test.py E501
scrapy/utils/url.py E501 F403 E128 F405
@ -181,7 +181,7 @@ flake8-ignore =
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E701 E126 E123
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
@ -189,10 +189,10 @@ flake8-ignore =
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E128 E127
tests/test_downloadermiddleware_retry.py E501 E128 E251 E126
tests/test_downloadermiddleware_retry.py E501 E128 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128 E124
@ -204,15 +204,15 @@ flake8-ignore =
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E128 E265
tests/test_item.py E701 E128 F841
tests/test_item.py E128 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E501 E731 E741 E128 E117 E241
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
tests/test_pipeline_files.py E501 E272
tests/test_pipeline_crawl.py E501 E128 E126
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E731 E128 E502
tests/test_proxy_connect.py E501 E741
@ -227,7 +227,7 @@ flake8-ignore =
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E128
tests/test_utils_curl.py E501
@ -237,13 +237,13 @@ flake8-ignore =
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E241
tests/test_utils_log.py E741
tests/test_utils_python.py E501 E731 E701
tests/test_utils_python.py E501 E731
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_url.py E501 E127 E211 E125 E501 E241 E126 E123
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128

6
scrapy/utils/iterators.py
View File

@ -101,8 +101,10 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
lines = StringIO(_body_or_str(obj, unicode=True))
kwargs = {}
if delimiter: kwargs["delimiter"] = delimiter
if quotechar: kwargs["quotechar"] = quotechar
if delimiter:
kwargs["delimiter"] = delimiter
if quotechar:
kwargs["quotechar"] = quotechar
csv_r = csv.reader(lines, **kwargs)
if not headers:

2
tests/test_downloadermiddleware_httpcompression.py
View File

@ -245,7 +245,7 @@ class HttpCompressionTest(TestCase):
response.headers['Content-Type'] = 'application/gzip'
request = response.request
request.method = 'HEAD'
response = response.replace(body = None)
response = response.replace(body=None)
newresponse = self.mw.process_response(request, response, self.spider)
self.assertIs(newresponse, response)
self.assertEqual(response.body, b'')

21
tests/test_item.py
View File

@ -149,13 +149,15 @@ class ItemTest(unittest.TestCase):
fields = {'load': Field(default='A')}
save = Field(default='A')
class B(A): pass
class B(A):
pass
class C(Item):
fields = {'load': Field(default='C')}
save = Field(default='C')
class D(B, C): pass
class D(B, C):
pass
item = D(save='X', load='Y')
self.assertEqual(item['save'], 'X')
@ -164,7 +166,8 @@ class ItemTest(unittest.TestCase):
'save': {'default': 'A'}})
# D class inverted
class E(C, B): pass
class E(C, B):
pass
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
@ -177,7 +180,8 @@ class ItemTest(unittest.TestCase):
save = Field(default='A')
load = Field(default='A')
class B(A): pass
class B(A):
pass
class C(A):
fields = {'update': Field(default='C')}
@ -206,14 +210,16 @@ class ItemTest(unittest.TestCase):
fields = {'load': Field(default='A')}
save = Field(default='A')
class B(A): pass
class B(A):
pass
class C(object):
fields = {'load': Field(default='C')}
not_allowed = Field(default='not_allowed')
save = Field(default='C')
class D(B, C): pass
class D(B, C):
pass
self.assertRaises(KeyError, D, not_allowed='value')
self.assertEqual(D(save='X')['save'], 'X')
@ -221,7 +227,8 @@ class ItemTest(unittest.TestCase):
'load': {'default': 'A'}})
# D class inverted
class E(C, B): pass
class E(C, B):
pass
self.assertRaises(KeyError, E, not_allowed='value')
self.assertEqual(E(save='X')['save'], 'X')

12
tests/test_pipeline_crawl.py
View File

@ -26,10 +26,9 @@ class MediaDownloadSpider(SimpleSpider):
self.media_key: [],
self.media_urls_key: [
self._process_url(response.urljoin(href))
for href in response.xpath('''
//table[thead/tr/th="Filename"]
/tbody//a/@href
''').getall()],
for href in response.xpath(
'//table[thead/tr/th="Filename"]/tbody//a/@href'
).getall()],
}
yield item
@ -99,8 +98,9 @@ class FileDownloadCrawlTestCase(TestCase):
if self.expected_checksums is not None:
checksums = set(
i['checksum']
for item in items
for i in item[self.media_key])
for item in items
for i in item[self.media_key]
)
self.assertEqual(checksums, self.expected_checksums)
# check that the image files where actually written to the media store

2
tests/test_pipeline_files.py
View File

@ -272,7 +272,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(prefix=prefix)
user_pipeline = pipeline_cls.from_settings(Settings(settings))
for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
custom_value = settings.get(prefix + "_" + settings_attr)
self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr])
self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)

4
tests/test_squeues.py
View File

@ -36,7 +36,9 @@ def nonserializable_object_test(self):
self.assertRaises(ValueError, q.push, lambda x: x)
else:
# Use a different unpickleable object
class A(object): pass
class A(object):
pass
a = A()
a.__reduce__ = a.__reduce_ex__ = None
self.assertRaises(ValueError, q.push, a)

4
tests/test_utils_python.py
View File

@ -153,7 +153,9 @@ class UtilsPythonTestCase(unittest.TestCase):
self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
def test_weakkeycache(self):
class _Weakme(object): pass
class _Weakme(object):
pass
_values = count()
wk = WeakKeyCache(lambda k: next(_values))
k = _Weakme()

6
tests/test_utils_url.py
View File

@ -202,7 +202,7 @@ def create_skipped_scheme_t(args):
return do_expected
for k, args in enumerate ([
for k, args in enumerate([
('/index', 'file://'),
('/index.html', 'file://'),
('./index.html', 'file://'),
@ -230,7 +230,7 @@ for k, args in enumerate ([
], start=1):
t_method = create_guess_scheme_t(args)
t_method.__name__ = 'test_uri_%03d' % k
setattr (GuessSchemeTest, t_method.__name__, t_method)
setattr(GuessSchemeTest, t_method.__name__, t_method)
# TODO: the following tests do not pass with current implementation
for k, args in enumerate([
@ -239,7 +239,7 @@ for k, args in enumerate([
], start=1):
t_method = create_skipped_scheme_t(args)
t_method.__name__ = 'test_uri_skipped_%03d' % k
setattr (GuessSchemeTest, t_method.__name__, t_method)
setattr(GuessSchemeTest, t_method.__name__, t_method)
class StripUrl(unittest.TestCase):