RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pass args/kwargs in S3DownloadHandler.from_crawler, update tests

This commit is contained in:
Eugenio Lacuesta 2019-11-05 16:18:42 -03:00
parent 3d77f74e40
commit e43f37fff3
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 13 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/core/downloader/handlers/s3.py
View File

@ -69,8 +69,8 @@ class S3DownloadHandler(object):
self._download_http = httpdownloadhandler(crawler).download_request
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def from_crawler(cls, crawler, *args, **kwargs):
return cls(crawler, *args, **kwargs)
def download_request(self, request, spider):
p = urlparse_cached(request)

22
tests/test_downloader_handlers.py
View File

@ -250,7 +250,7 @@ class HttpTestCase(unittest.TestCase):
else:
self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
self.portno = self.port.getHost().port
self.download_handler = self.download_handler_cls(get_crawler())
self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@ -490,7 +490,7 @@ class Http11TestCase(HttpTestCase):
return self.test_download_broken_content_allow_data_loss('broken-chunked')
def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):
download_handler = self.download_handler_cls(
download_handler = self.download_handler_cls.from_crawler(
get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})
)
request = Request(self.getURL(url))
@ -510,7 +510,7 @@ class Https11TestCase(Http11TestCase):
@defer.inlineCallbacks
def test_tls_logging(self):
download_handler = self.download_handler_cls(
download_handler = self.download_handler_cls.from_crawler(
get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})
)
try:
@ -579,7 +579,7 @@ class Https11CustomCiphers(unittest.TestCase):
0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
interface=self.host)
self.portno = self.port.getHost().port
self.download_handler = self.download_handler_cls(
self.download_handler = self.download_handler_cls.from_crawler(
get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
)
self.download_request = self.download_handler.download_request
@ -677,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper = WrappingFactory(site)
self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
self.download_handler = self.download_handler_cls(get_crawler())
self.download_handler = self.download_handler_cls.from_crawler(get_crawler())
self.download_request = self.download_handler.download_request
@defer.inlineCallbacks
@ -750,7 +750,7 @@ class S3AnonTestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
self.s3reqh = S3DownloadHandler(
self.s3reqh = S3DownloadHandler.from_crawler(
crawler=get_crawler(),
httpdownloadhandler=HttpDownloadHandlerMock,
#anon=True, # is implicit
@ -779,10 +779,10 @@ class S3TestCase(unittest.TestCase):
def setUp(self):
skip_if_no_boto()
s3reqh = S3DownloadHandler(
get_crawler(),
self.AWS_ACCESS_KEY_ID,
self.AWS_SECRET_ACCESS_KEY,
s3reqh = S3DownloadHandler.from_crawler(
crawler=get_crawler(),
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
httpdownloadhandler=HttpDownloadHandlerMock,
)
self.download_request = s3reqh.download_request
@ -804,7 +804,7 @@ class S3TestCase(unittest.TestCase):
def test_extra_kw(self):
try:
S3DownloadHandler(get_crawler(), extra_kw=True)
S3DownloadHandler.from_crawler(get_crawler(), extra_kw=True)
except Exception as e:
self.assertIsInstance(e, (TypeError, NotConfigured))
else: