RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4538 from elacuesta/flake8-remove-e127 

Flake8: remove E127
This commit is contained in:
Andrey Rahmatullin 2020-05-06 21:34:24 +05:00 committed by GitHub
parent 628c4a5319 fe0c582ee0
commit 977ce9f7a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 114 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pytest.ini
View File

@ -41,13 +41,13 @@ flake8-ignore =
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/startproject.py E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127
scrapy/core/engine.py E501 E128
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128
scrapy/core/spidermw.py E501 E126
@ -57,7 +57,7 @@ flake8-ignore =
scrapy/core/downloader/tls.py E501
scrapy/core/downloader/webclient.py E501 E128 E126
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
scrapy/core/downloader/handlers/ftp.py E501 E128
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
scrapy/core/downloader/handlers/s3.py E501 E128 E126
@ -124,7 +124,7 @@ flake8-ignore =
scrapy/utils/datatypes.py E501
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127
scrapy/utils/deprecate.py E501
scrapy/utils/gz.py E501
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
@ -137,7 +137,7 @@ flake8-ignore =
scrapy/utils/python.py E501
scrapy/utils/reactor.py E501
scrapy/utils/reqser.py E501
scrapy/utils/request.py E127 E501
scrapy/utils/request.py E501
scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
@ -171,8 +171,8 @@ flake8-ignore =
tests/__init__.py E402 E501
tests/mockserver.py E401 E501 E126 E123
tests/pipelines.py F841
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
tests/spiders.py E501
tests/test_closespider.py E501
tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128
tests/test_command_shell.py E501 E128
@ -181,17 +181,17 @@ flake8-ignore =
tests/test_crawl.py E501 E741
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
tests/test_downloader_handlers.py E124 E128 E501 E126 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
tests/test_downloadermiddleware_httpcache.py E501
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
tests/test_downloadermiddleware_decompression.py E501
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E128 E127
tests/test_downloadermiddleware_redirect.py E501 E128
tests/test_downloadermiddleware_retry.py E501 E128 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
@ -202,7 +202,7 @@ flake8-ignore =
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_request.py E402 E501 E128 E128 E126 E123
tests/test_http_response.py E501 E128
tests/test_item.py E128 F841
tests/test_link.py E501
@ -220,10 +220,10 @@ flake8-ignore =
tests/test_responsetypes.py E501
tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_selector.py E501
tests/test_spider.py E501
tests/test_spidermiddleware.py E501
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_httperror.py E128 E501 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
@ -243,7 +243,7 @@ flake8-ignore =
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
tests/test_utils_url.py E501 E125 E501 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E123 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128

11
scrapy/core/downloader/handlers/ftp.py
View File

@ -94,11 +94,12 @@ class FTPDownloadHandler:
def gotClient(self, client, request, filepath):
self.client = client
protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
return client.retrieveFile(filepath, protocol)\
.addCallbacks(callback=self._build_response,
callbackArgs=(request, protocol),
errback=self._failed,
errbackArgs=(request,))
return client.retrieveFile(filepath, protocol).addCallbacks(
callback=self._build_response,
callbackArgs=(request, protocol),
errback=self._failed,
errbackArgs=(request,),
)
def _build_response(self, result, request, protocol):
self.result = result

3
scrapy/core/engine.py
View File

@ -230,8 +230,7 @@ class ExecutionEngine:
def _downloaded(self, response, slot, request, spider):
slot.remove_request(request)
return self.download(response, spider) \
if isinstance(response, Request) else response
return self.download(response, spider) if isinstance(response, Request) else response
def _download(self, request, spider):
slot = self.slot

21
scrapy/utils/deprecate.py
View File

@ -15,16 +15,17 @@ def attribute(obj, oldattr, newattr, version='0.12'):
stacklevel=3)
def create_deprecated_class(name, new_class, clsdict=None,
warn_category=ScrapyDeprecationWarning,
warn_once=True,
old_class_path=None,
new_class_path=None,
subclass_warn_message="{cls} inherits from "
"deprecated class {old}, please inherit "
"from {new}.",
instance_warn_message="{cls} is deprecated, "
"instantiate {new} instead."):
def create_deprecated_class(
name,
new_class,
clsdict=None,
warn_category=ScrapyDeprecationWarning,
warn_once=True,
old_class_path=None,
new_class_path=None,
subclass_warn_message="{cls} inherits from deprecated class {old}, please inherit from {new}.",
instance_warn_message="{cls} is deprecated, instantiate {new} instead."
):
"""
Return a "deprecated" class that causes its subclasses to issue a warning.
Subclasses of ``new_class`` are considered subclasses of this class.

3
scrapy/utils/request.py
View File

@ -50,8 +50,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
"""
if include_headers:
include_headers = tuple(to_bytes(h.lower())
for h in sorted(include_headers))
include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
cache = _fingerprint_cache.setdefault(request, {})
cache_key = (include_headers, keep_fragments)
if cache_key not in cache:

3
tests/spiders.py
View File

@ -184,8 +184,7 @@ class BrokenStartRequestsSpider(FollowAllSpider):
if self.fail_yielding:
2 / 0
assert self.seedsseen, \
'All start requests consumed before any download happened'
assert self.seedsseen, 'All start requests consumed before any download happened'
def parse(self, response):
self.seedsseen.append(response.meta.get('seed'))

3
tests/test_closespider.py
View File

@ -41,8 +41,7 @@ class TestCloseSpider(TestCase):
yield crawler.crawl(total=1000000, mockserver=self.mockserver)
reason = crawler.spider.meta['close_reason']
self.assertEqual(reason, 'closespider_errorcount')
key = 'spider_exceptions/{name}'\
.format(name=crawler.spider.exception_cls.__name__)
key = 'spider_exceptions/{name}'.format(name=crawler.spider.exception_cls.__name__)
errorcount = crawler.stats.get_value(key)
self.assertTrue(errorcount >= close_on)

9
tests/test_downloader_handlers.py
View File

@ -1090,8 +1090,7 @@ class DataURITestCase(unittest.TestCase):
def test_default_mediatype_encoding(self):
def _test(response):
self.assertEqual(response.text, 'A brief note')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "US-ASCII")
request = Request("data:,A%20brief%20note")
@ -1100,8 +1099,7 @@ class DataURITestCase(unittest.TestCase):
def test_default_mediatype(self):
def _test(response):
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "iso-8859-7")
request = Request("data:;charset=iso-8859-7,%be%d3%be")
@ -1119,8 +1117,7 @@ class DataURITestCase(unittest.TestCase):
def test_mediatype_parameters(self):
def _test(response):
self.assertEqual(response.text, u'\u038e\u03a3\u038e')
self.assertEqual(type(response),
responsetypes.from_mimetype("text/plain"))
self.assertEqual(type(response), responsetypes.from_mimetype("text/plain"))
self.assertEqual(response.encoding, "utf-8")
request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'

4
tests/test_downloadermiddleware_decompression.py
View File

@ -28,8 +28,8 @@ class DecompressionMiddlewareTest(TestCase):
for fmt in self.test_formats:
rsp = self.test_responses[fmt]
new = self.mw.process_response(None, rsp, self.spider)
assert isinstance(new, XmlResponse), \
'Failed %s, response type %s' % (fmt, type(new).__name__)
error_msg = 'Failed %s, response type %s' % (fmt, type(new).__name__)
assert isinstance(new, XmlResponse), error_msg
assert_samelines(self, new.body, self.uncompressed_body, fmt)
def test_plain_response(self):

3
tests/test_downloadermiddleware_redirect.py
View File

@ -181,8 +181,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
rsp = Response(url, headers={'Location': url2}, status=301, request=req)
r = self.mw.process_response(req, rsp, self.spider)
self.assertIs(r, rsp)
_test_passthrough(Request(url, meta={'handle_httpstatus_list':
[404, 301, 302]}))
_test_passthrough(Request(url, meta={'handle_httpstatus_list': [404, 301, 302]}))
_test_passthrough(Request(url, meta={'handle_httpstatus_all': True}))
def test_latin1_location(self):

6
tests/test_http_request.py
View File

@ -399,8 +399,7 @@ class FormRequestTest(RequestTest):
def test_custom_encoding_bytes(self):
data = {b'\xb5 one': b'two', b'price': b'\xa3 100'}
r2 = self.request_class("http://www.example.com", formdata=data,
encoding='latin1')
r2 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
self.assertEqual(r2.method, 'POST')
self.assertEqual(r2.encoding, 'latin1')
self.assertQueryEqual(r2.body, b'price=%A3+100&%B5+one=two')
@ -408,8 +407,7 @@ class FormRequestTest(RequestTest):
def test_custom_encoding_textual_data(self):
data = {'price': u'£ 100'}
r3 = self.request_class("http://www.example.com", formdata=data,
encoding='latin1')
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
self.assertEqual(r3.encoding, 'latin1')
self.assertEqual(r3.body, b'price=%A3+100')

3
tests/test_selector.py
View File

@ -67,8 +67,7 @@ class SelectorTestCase(unittest.TestCase):
headers = {'Content-Type': ['text/html; charset=utf-8']}
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
x = Selector(response)
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
[u'\xa3'])
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3'])
def test_badly_encoded_body(self):
# \xe9 alone isn't valid utf8 sequence

6
tests/test_spidermiddleware_httperror.py
View File

@ -111,8 +111,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()
@ -146,8 +145,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
res404 = self.res404.copy()
res404.request = request
res402 = self.res402.copy()

120
tests/test_utils_url.py
View File

@ -77,108 +77,124 @@ class UrlUtilsTest(unittest.TestCase):
class AddHttpIfNoScheme(unittest.TestCase):
def test_add_scheme(self):
self.assertEqual(add_http_if_no_scheme('www.example.com'),
'http://www.example.com')
self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com')
def test_without_subdomain(self):
self.assertEqual(add_http_if_no_scheme('example.com'),
'http://example.com')
self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com')
def test_path(self):
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
self.assertEqual(
add_http_if_no_scheme('www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
def test_port(self):
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
'http://www.example.com:80')
self.assertEqual(
add_http_if_no_scheme('www.example.com:80'),
'http://www.example.com:80')
def test_fragment(self):
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
self.assertEqual(
add_http_if_no_scheme('www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
def test_query(self):
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
self.assertEqual(
add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
def test_username_password(self):
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
'http://username:password@www.example.com')
self.assertEqual(
add_http_if_no_scheme('username:password@www.example.com'),
'http://username:password@www.example.com')
def test_complete_url(self):
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
self.assertEqual(
add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
def test_preserve_http(self):
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
'http://www.example.com')
self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com')
def test_preserve_http_without_subdomain(self):
self.assertEqual(add_http_if_no_scheme('http://example.com'),
'http://example.com')
self.assertEqual(
add_http_if_no_scheme('http://example.com'),
'http://example.com')
def test_preserve_http_path(self):
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
self.assertEqual(
add_http_if_no_scheme('http://www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
def test_preserve_http_port(self):
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
'http://www.example.com:80')
self.assertEqual(
add_http_if_no_scheme('http://www.example.com:80'),
'http://www.example.com:80')
def test_preserve_http_fragment(self):
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
self.assertEqual(
add_http_if_no_scheme('http://www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
def test_preserve_http_query(self):
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
self.assertEqual(
add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
def test_preserve_http_username_password(self):
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
'http://username:password@www.example.com')
self.assertEqual(
add_http_if_no_scheme('http://username:password@www.example.com'),
'http://username:password@www.example.com')
def test_preserve_http_complete_url(self):
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
self.assertEqual(
add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
def test_protocol_relative(self):
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
'http://www.example.com')
self.assertEqual(
add_http_if_no_scheme('//www.example.com'), 'http://www.example.com')
def test_protocol_relative_without_subdomain(self):
self.assertEqual(add_http_if_no_scheme('//example.com'),
'http://example.com')
self.assertEqual(
add_http_if_no_scheme('//example.com'), 'http://example.com')
def test_protocol_relative_path(self):
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
self.assertEqual(
add_http_if_no_scheme('//www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
def test_protocol_relative_port(self):
self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
'http://www.example.com:80')
self.assertEqual(
add_http_if_no_scheme('//www.example.com:80'),
'http://www.example.com:80')
def test_protocol_relative_fragment(self):
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
self.assertEqual(
add_http_if_no_scheme('//www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
def test_protocol_relative_query(self):
self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
self.assertEqual(
add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
def test_protocol_relative_username_password(self):
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
'http://username:password@www.example.com')
self.assertEqual(
add_http_if_no_scheme('//username:password@www.example.com'),
'http://username:password@www.example.com')
def test_protocol_relative_complete_url(self):
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
self.assertEqual(
add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
def test_preserve_https(self):
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
'https://www.example.com')
self.assertEqual(
add_http_if_no_scheme('https://www.example.com'),
'https://www.example.com')
def test_preserve_ftp(self):
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
'ftp://www.example.com')
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), 'ftp://www.example.com')
class GuessSchemeTest(unittest.TestCase):