RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Flake8: remove W504 code (#4525) 

Co-authored-by: Mikhail Korobov <kmike84@gmail.com>
This commit is contained in:
Eugenio Lacuesta 2020-05-05 19:14:48 -03:00 committed by GitHub
parent 8f72b70f06
commit 17c0cf64ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 35 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
pytest.ini
View File

@ -44,12 +44,12 @@ flake8-ignore =
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/__init__.py E501
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128 W504
scrapy/core/scraper.py E501 E128
scrapy/core/spidermw.py E501 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
@ -68,7 +68,7 @@ flake8-ignore =
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/redirect.py E501
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
@ -79,7 +79,7 @@ flake8-ignore =
scrapy/extensions/httpcache.py E128 E501
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/telnet.py E501
scrapy/extensions/throttle.py E501
# scrapy/http
scrapy/http/common.py E501
@ -90,7 +90,7 @@ flake8-ignore =
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E501 E402 W504
scrapy/linkextractors/__init__.py E501 E402
scrapy/linkextractors/lxmlhtml.py E501
# scrapy/loader
scrapy/loader/__init__.py E501 E128
@ -110,7 +110,7 @@ flake8-ignore =
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
scrapy/spidermiddlewares/referer.py E501 E129 W504
scrapy/spidermiddlewares/referer.py E501 E129
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
@ -125,7 +125,7 @@ flake8-ignore =
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127
scrapy/utils/gz.py E501 W504
scrapy/utils/gz.py E501
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501
@ -234,7 +234,7 @@ flake8-ignore =
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
tests/test_utils_http.py E501 E128 W504
tests/test_utils_http.py E501 E128
tests/test_utils_iterators.py E501 E128 E129
tests/test_utils_log.py E741
tests/test_utils_python.py E501

4
scrapy/contracts/__init__.py
View File

@ -17,10 +17,10 @@ class ContractsManager:
self.contracts[contract.name] = contract
def tested_methods_from_spidercls(self, spidercls):
is_method = re.compile(r"^\s*@", re.MULTILINE).search
methods = []
for key, value in getmembers(spidercls):
if (callable(value) and value.__doc__ and
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
if callable(value) and value.__doc__ and is_method(value.__doc__):
methods.append(key)
return methods

11
scrapy/downloadermiddlewares/redirect.py
View File

@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
Handle redirection of requests based on response status
and meta-refresh html tag.
"""
def process_response(self, request, response, spider):
if (request.meta.get('dont_redirect', False) or
response.status in getattr(spider, 'handle_httpstatus_list', []) or
response.status in request.meta.get('handle_httpstatus_list', []) or
request.meta.get('handle_httpstatus_all', False)):
if (
request.meta.get('dont_redirect', False)
or response.status in getattr(spider, 'handle_httpstatus_list', [])
or response.status in request.meta.get('handle_httpstatus_list', [])
or request.meta.get('handle_httpstatus_all', False)
):
return response
allowed_status = (301, 302, 303, 307, 308)

6
scrapy/extensions/telnet.py
View File

@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
"""An implementation of IPortal"""
@defers
def login(self_, credentials, mind, *interfaces):
if not (credentials.username == self.username.encode('utf8') and
credentials.checkPassword(self.password.encode('utf8'))):
if not (
credentials.username == self.username.encode('utf8')
and credentials.checkPassword(self.password.encode('utf8'))
):
raise ValueError("Invalid credentials")
protocol = telnet.TelnetBootstrapProtocol(

3
scrapy/linkextractors/__init__.py
View File

@ -61,8 +61,7 @@ class FilteringLinkExtractor:
def __new__(cls, *args, **kwargs):
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
if (issubclass(cls, FilteringLinkExtractor) and
not issubclass(cls, LxmlLinkExtractor)):
if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)

14
scrapy/spidermiddlewares/referer.py
View File

@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
name = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
if ((self.tls_protected(response_url) and
self.potentially_trustworthy(request_url))
or not self.tls_protected(response_url)):
if (
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)
@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
elif ((self.tls_protected(response_url) and
self.potentially_trustworthy(request_url))
or not self.tls_protected(response_url)):
elif (
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)

3
scrapy/utils/gz.py
View File

@ -52,8 +52,7 @@ def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
ctype = response.headers.get('Content-Type', b'')
cenc = response.headers.get('Content-Encoding', b'').lower()
return (_is_gzipped(ctype) or
(_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
def gzip_magic_number(response):

8
tests/test_utils_http.py
View File

@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
chunked_body += "8\r\n" + "sequence\r\n"
chunked_body += "0\r\n\r\n"
body = decode_chunked_transfer(chunked_body)
self.assertEqual(body,
"This is the data in the first chunk\r\n" +
"and this is the second one\r\n" +
"consequence")
self.assertEqual(
body,
"This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
)