RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into fix-pycodestyle-e731

This commit is contained in:
Eugenio Lacuesta 2020-04-10 14:32:30 -03:00
parent c887fe37ad 39b01b6892
commit 2265c1fda9
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
9 changed files with 39 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pytest.ini
View File

@ -36,24 +36,24 @@ flake8-ignore =
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/genspider.py E128 E501
scrapy/commands/parse.py E128 E501
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/shell.py E128 E501
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127 E502
scrapy/core/engine.py E501 E128 E127
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128 W504
scrapy/core/spidermw.py E501 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/middleware.py E501
scrapy/core/downloader/tls.py E501 E241
scrapy/core/downloader/webclient.py E501 E128 E126
scrapy/core/downloader/handlers/__init__.py E501
@ -124,7 +124,7 @@ flake8-ignore =
scrapy/utils/datatypes.py E501
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/deprecate.py E128 E501 E127
scrapy/utils/gz.py E501 W504
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
@ -156,7 +156,7 @@ flake8-ignore =
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/mail.py E402 E128 E501
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/resolver.py E501
@ -214,7 +214,7 @@ flake8-ignore =
tests/test_pipeline_crawl.py E501 E128 E126
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E128 E502
tests/test_pipeline_media.py E501 E741 E128
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501

7
scrapy/commands/genspider.py
View File

@ -90,8 +90,7 @@ class Command(ScrapyCommand):
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() \
for s in module.split('_'))
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -102,8 +101,8 @@ class Command(ScrapyCommand):
spider_file = "%s.py" % join(spiders_dir, module)
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r " % (name, \
template_name), end=('' if spiders_module else '\n'))
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))

2
scrapy/commands/shell.py
View File

@ -37,7 +37,7 @@ class Command(ScrapyCommand):
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):

14
scrapy/core/downloader/handlers/http11.py
View File

@ -341,20 +341,6 @@ class ScrapyAgent:
headers.removeHeader(b'Proxy-Authorization')
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
elif method == b'POST':
# Setting Content-Length: 0 even for POST requests is not a
# MUST per HTTP RFCs, but it's common behavior, and some
# servers require this, otherwise returning HTTP 411 Length required
#
# RFC 7230#section-3.3.2:
# "a Content-Length header field is normally sent in a POST
# request even when the value is 0 (indicating an empty payload body)."
#
# Twisted < 17 will not add "Content-Length: 0" by itself;
# Twisted >= 17 fixes this;
# Using a producer with an empty-string sends `0` as Content-Length
# for all versions of Twisted.
bodyproducer = _RequestBodyProducer(b'')
else:
bodyproducer = None
start_time = time()

18
scrapy/core/downloader/middleware.py
View File

@ -35,8 +35,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, response.__class__.__name__))
raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, response.__class__.__name__)
)
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@ -50,8 +52,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_response']:
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
)
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@ -62,8 +66,10 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
)
if response:
defer.returnValue(response)
defer.returnValue(_failure)

5
scrapy/core/engine.py
View File

@ -277,10 +277,9 @@ class ExecutionEngine:
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = self.signals.send_catch_log(signal=signals.spider_idle, \
res = self.signals.send_catch_log(signal=signals.spider_idle,
spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
for _, x in res):
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
return
if self.spider_is_idle(spider):

4
scrapy/mail.py
View File

@ -115,8 +115,8 @@ class MailSender:
from twisted.mail.smtp import ESMTPSenderFactory
msg = BytesIO(msg)
d = defer.Deferred()
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
requireTransportSecurity=self.smtptls)
factory.noisy = False

15
scrapy/utils/deprecate.py
View File

@ -7,9 +7,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning
def attribute(obj, oldattr, newattr, version='0.12'):
cname = obj.__class__.__name__
warnings.warn("%s.%s attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead" % \
(cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)
warnings.warn(
"%s.%s attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead"
% (cname, oldattr, version, cname, newattr),
ScrapyDeprecationWarning,
stacklevel=3)
def create_deprecated_class(name, new_class, clsdict=None,
@ -17,10 +20,10 @@ def create_deprecated_class(name, new_class, clsdict=None,
warn_once=True,
old_class_path=None,
new_class_path=None,
subclass_warn_message="{cls} inherits from "\
"deprecated class {old}, please inherit "\
subclass_warn_message="{cls} inherits from "
"deprecated class {old}, please inherit "
"from {new}.",
instance_warn_message="{cls} is deprecated, "\
instance_warn_message="{cls} is deprecated, "
"instantiate {new} instead."):
"""
Return a "deprecated" class that causes its subclasses to issue a warning.

5
tests/test_pipeline_media.py
View File

@ -331,8 +331,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, 'ITSME')])
self.assertEqual(self.pipe._mockcalled, \
['get_media_requests', 'media_to_download', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'item_completed'])
class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):