diff --git a/pytest.ini b/pytest.ini index d96ac9684..7c6ee07d6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -47,20 +47,20 @@ flake8-ignore = scrapy/contracts/__init__.py E501 scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E501 E128 + scrapy/core/engine.py E501 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E128 + scrapy/core/scraper.py E501 scrapy/core/spidermw.py E501 scrapy/core/downloader/__init__.py E501 - scrapy/core/downloader/contextfactory.py E501 E128 + scrapy/core/downloader/contextfactory.py E501 scrapy/core/downloader/middleware.py E501 scrapy/core/downloader/tls.py E501 - scrapy/core/downloader/webclient.py E501 E128 + scrapy/core/downloader/webclient.py E501 scrapy/core/downloader/handlers/__init__.py E501 - scrapy/core/downloader/handlers/ftp.py E501 E128 + scrapy/core/downloader/handlers/ftp.py E501 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 E128 + scrapy/core/downloader/handlers/s3.py E501 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 scrapy/downloadermiddlewares/decompression.py E501 diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index ab73e12c8..452242d47 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -46,11 +46,12 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # # * getattr() for `_ssl_method` attribute for context factories # not calling super(..., self).__init__ - return CertificateOptions(verify=False, - method=getattr(self, 'method', - getattr(self, '_ssl_method', None)), - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers) + return CertificateOptions( + verify=False, + method=getattr(self, 'method', getattr(self, '_ssl_method', None)), + fixBrokenPeers=True, + acceptableCiphers=self.tls_ciphers, + ) # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 94b55c347..3ef129587 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -86,10 +86,9 @@ class FTPDownloadHandler: password = request.meta.get("ftp_password", self.default_password) passive_mode = 1 if bool(request.meta.get("ftp_passive", self.passive_mode)) else 0 - creator = ClientCreator(reactor, FTPClient, user, password, - passive=passive_mode) - return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient, - request, unquote(parsed_url.path)) + creator = ClientCreator(reactor, FTPClient, user, password, passive=passive_mode) + dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21) + return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path)) def gotClient(self, client, request, filepath): self.client = client diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 324d21716..de0da4b70 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -217,11 +217,9 @@ class ExecutionEngine: self.slot.nextcall.schedule() def schedule(self, request, spider): - self.signals.send_catch_log(signal=signals.request_scheduled, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_scheduled, request=request, spider=spider) if not self.slot.scheduler.enqueue_request(request): - self.signals.send_catch_log(signal=signals.request_dropped, - request=request, spider=spider) + self.signals.send_catch_log(signals.request_dropped, request=request, spider=spider) def download(self, request, spider): d = self._download(request, spider) @@ -247,8 +245,8 @@ class ExecutionEngine: logkws = self.logformatter.crawled(request, response, spider) if logkws is not None: logger.log(*logformatter_adapter(logkws), extra={'spider': spider}) - self.signals.send_catch_log(signal=signals.response_received, - response=response, request=request, spider=spider) + self.signals.send_catch_log(signals.response_received, + response=response, request=request, spider=spider) return response def _on_complete(_): @@ -286,8 +284,7 @@ class ExecutionEngine: next loop and this function is guaranteed to be called (at least) once again for this spider. """ - res = self.signals.send_catch_log(signal=signals.spider_idle, - spider=spider, dont_log=DontCloseSpider) + res = self.signals.send_catch_log(signals.spider_idle, spider=spider, dont_log=DontCloseSpider) if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res): return