From dd1398b280334b1014d2dbed009331ceed7c4fb1 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 3 Sep 2012 16:43:51 -0300 Subject: [PATCH] removed another instance of scrapy.conf.settings singleton, this time from download handlers --- docs/news.rst | 1 + scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/__init__.py | 5 ++--- scrapy/core/downloader/handlers/file.py | 3 +++ scrapy/core/downloader/handlers/http.py | 14 +++++--------- scrapy/core/downloader/handlers/s3.py | 5 ++--- scrapy/tests/test_downloader_handlers.py | 12 ++++++++---- 7 files changed, 22 insertions(+), 20 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 1517ed276..1abb88fcf 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -30,6 +30,7 @@ Scrapy changes: - removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library - promoted :ref:`topics-djangoitem` to main contrib - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`) +- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor Scrapyd changes: diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 456b0cce1..042f78075 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -70,7 +70,7 @@ class Downloader(object): self.signals = crawler.signals self.slots = {} self.active = set() - self.handlers = DownloadHandlers() + self.handlers = DownloadHandlers(crawler.settings) self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 69083b34c..90c557abf 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -2,13 +2,12 @@ from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached -from scrapy.conf import settings from scrapy.utils.misc import load_object class DownloadHandlers(object): - def __init__(self): + def __init__(self, settings): self._handlers = {} self._notconfigured = {} handlers = settings.get('DOWNLOAD_HANDLERS_BASE') @@ -16,7 +15,7 @@ class DownloadHandlers(object): for scheme, clspath in handlers.iteritems(): cls = load_object(clspath) try: - dh = cls() + dh = cls(settings) except NotConfigured, ex: self._notconfigured[scheme] = str(ex) else: diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index a63a5ffeb..85bad1583 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -4,6 +4,9 @@ from scrapy.utils.decorator import defers class FileDownloadHandler(object): + def __init__(self, settings): + pass + @defers def download_request(self, request, spider): filepath = file_uri_to_path(request.url) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index ff0beb321..f566d090b 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -4,23 +4,19 @@ from twisted.internet import reactor from scrapy.exceptions import NotSupported from scrapy.utils.misc import load_object -from scrapy.conf import settings from scrapy import optional_features ssl_supported = 'ssl' in optional_features -HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) -ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - - class HttpDownloadHandler(object): - def __init__(self, httpclientfactory=HTTPClientFactory): - self.httpclientfactory = httpclientfactory + def __init__(self, settings): + self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) + self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) def download_request(self, request, spider): """Return a deferred for the HTTP download""" - factory = self.httpclientfactory(request) + factory = self.HTTPClientFactory(request) self._connect(factory) return factory.deferred @@ -29,7 +25,7 @@ class HttpDownloadHandler(object): if factory.scheme == 'https': if ssl_supported: return reactor.connectSSL(host, port, factory, \ - ClientContextFactory()) + self.ClientContextFactory()) raise NotSupported("HTTPS not supported: install pyopenssl library") else: return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 9033126c6..4986464a6 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,7 +1,6 @@ from scrapy import optional_features from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached -from scrapy.conf import settings from .http import HttpDownloadHandler try: @@ -30,7 +29,7 @@ else: class S3DownloadHandler(object): - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, \ + def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ httpdownloadhandler=HttpDownloadHandler): if 'boto' not in optional_features: raise NotConfigured("missing boto library") @@ -44,7 +43,7 @@ class S3DownloadHandler(object): self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key) except Exception, ex: raise NotConfigured(str(ex)) - self._download_http = httpdownloadhandler().download_request + self._download_http = httpdownloadhandler(settings).download_request def download_request(self, request, spider): p = urlparse_cached(request) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index ae1fb3460..adf2d2433 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -15,6 +15,7 @@ from scrapy.core.downloader.handlers.http import HttpDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler from scrapy.spider import BaseSpider from scrapy.http import Request +from scrapy.settings import Settings from scrapy import optional_features @@ -25,7 +26,7 @@ class FileTestCase(unittest.TestCase): fd = open(self.tmpname + '^', 'w') fd.write('0123456789') fd.close() - self.download_request = FileDownloadHandler().download_request + self.download_request = FileDownloadHandler(Settings()).download_request def test_download(self): def _test(response): @@ -60,7 +61,7 @@ class HttpTestCase(unittest.TestCase): self.wrapper = WrappingFactory(self.site) self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port - self.download_request = HttpDownloadHandler().download_request + self.download_request = HttpDownloadHandler(Settings()).download_request def tearDown(self): return self.port.stopListening() @@ -148,7 +149,7 @@ class HttpProxyTestCase(unittest.TestCase): wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port - self.download_request = HttpDownloadHandler().download_request + self.download_request = HttpDownloadHandler(Settings()).download_request def tearDown(self): return self.port.stopListening() @@ -177,6 +178,9 @@ class HttpProxyTestCase(unittest.TestCase): class HttpDownloadHandlerMock(object): + def __init__(self, settings): + pass + def download_request(self, request, spider): return request @@ -191,7 +195,7 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' def setUp(self): - s3reqh = S3DownloadHandler(self.AWS_ACCESS_KEY_ID, \ + s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \ self.AWS_SECRET_ACCESS_KEY, \ httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request