removed another instance of scrapy.conf.settings singleton, this time from download handlers

docs/news.rst

@@ -30,6 +30,7 @@ Scrapy changes:
 - removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
 - promoted :ref:`topics-djangoitem` to main contrib
 - LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
+- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
 
 Scrapyd changes:
 

scrapy/core/downloader/__init__.py

@@ -70,7 +70,7 @@ class Downloader(object):
         self.signals = crawler.signals
         self.slots = {}
         self.active = set()
-        self.handlers = DownloadHandlers()
+        self.handlers = DownloadHandlers(crawler.settings)
         self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
         self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')

scrapy/core/downloader/handlers/__init__.py

@@ -2,13 +2,12 @@
 
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.conf import settings
 from scrapy.utils.misc import load_object
 
 
 class DownloadHandlers(object):
 
-    def __init__(self):
+    def __init__(self, settings):
         self._handlers = {}
         self._notconfigured = {}
         handlers = settings.get('DOWNLOAD_HANDLERS_BASE')
@@ -16,7 +15,7 @@ class DownloadHandlers(object):
         for scheme, clspath in handlers.iteritems():
             cls = load_object(clspath)
             try:
-                dh = cls()
+                dh = cls(settings)
             except NotConfigured, ex:
                 self._notconfigured[scheme] = str(ex)
             else:

scrapy/core/downloader/handlers/file.py

@@ -4,6 +4,9 @@ from scrapy.utils.decorator import defers
 
 class FileDownloadHandler(object):
 
+    def __init__(self, settings):
+        pass
+
     @defers
     def download_request(self, request, spider):
         filepath = file_uri_to_path(request.url)

scrapy/core/downloader/handlers/http.py

@@ -4,23 +4,19 @@ from twisted.internet import reactor
 
 from scrapy.exceptions import NotSupported
 from scrapy.utils.misc import load_object
-from scrapy.conf import settings
 from scrapy import optional_features
 
 ssl_supported = 'ssl' in optional_features
 
-HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
-ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-
-
 class HttpDownloadHandler(object):
 
-    def __init__(self, httpclientfactory=HTTPClientFactory):
-        self.httpclientfactory = httpclientfactory
+    def __init__(self, settings):
+        self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+        self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
 
     def download_request(self, request, spider):
         """Return a deferred for the HTTP download"""
-        factory = self.httpclientfactory(request)
+        factory = self.HTTPClientFactory(request)
         self._connect(factory)
         return factory.deferred
 
@@ -29,7 +25,7 @@ class HttpDownloadHandler(object):
         if factory.scheme == 'https':
             if ssl_supported:
                 return reactor.connectSSL(host, port, factory, \
-                        ClientContextFactory())
+                        self.ClientContextFactory())
             raise NotSupported("HTTPS not supported: install pyopenssl library")
         else:
             return reactor.connectTCP(host, port, factory)

scrapy/core/downloader/handlers/s3.py

@@ -1,7 +1,6 @@
 from scrapy import optional_features
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.conf import settings
 from .http import HttpDownloadHandler
 
 try:
@@ -30,7 +29,7 @@ else:
 
 class S3DownloadHandler(object):
 
-    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, \
+    def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
             httpdownloadhandler=HttpDownloadHandler):
         if 'boto' not in optional_features:
             raise NotConfigured("missing boto library")
@@ -44,7 +43,7 @@ class S3DownloadHandler(object):
             self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key)
         except Exception, ex:
             raise NotConfigured(str(ex))
-        self._download_http = httpdownloadhandler().download_request
+        self._download_http = httpdownloadhandler(settings).download_request
 
     def download_request(self, request, spider):
         p = urlparse_cached(request)

scrapy/tests/test_downloader_handlers.py

@@ -15,6 +15,7 @@ from scrapy.core.downloader.handlers.http import HttpDownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
+from scrapy.settings import Settings
 from scrapy import optional_features
 
 
@@ -25,7 +26,7 @@ class FileTestCase(unittest.TestCase):
         fd = open(self.tmpname + '^', 'w')
         fd.write('0123456789')
         fd.close()
-        self.download_request = FileDownloadHandler().download_request
+        self.download_request = FileDownloadHandler(Settings()).download_request
 
     def test_download(self):
         def _test(response):
@@ -60,7 +61,7 @@ class HttpTestCase(unittest.TestCase):
         self.wrapper = WrappingFactory(self.site)
         self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_request = HttpDownloadHandler().download_request
+        self.download_request = HttpDownloadHandler(Settings()).download_request
 
     def tearDown(self):
         return self.port.stopListening()
@@ -148,7 +149,7 @@ class HttpProxyTestCase(unittest.TestCase):
         wrapper = WrappingFactory(site)
         self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
         self.portno = self.port.getHost().port
-        self.download_request = HttpDownloadHandler().download_request
+        self.download_request = HttpDownloadHandler(Settings()).download_request
 
     def tearDown(self):
         return self.port.stopListening()
@@ -177,6 +178,9 @@ class HttpProxyTestCase(unittest.TestCase):
 
 
 class HttpDownloadHandlerMock(object):
+    def __init__(self, settings):
+        pass
+
     def download_request(self, request, spider):
         return request
 
@@ -191,7 +195,7 @@ class S3TestCase(unittest.TestCase):
     AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
 
     def setUp(self):
-        s3reqh = S3DownloadHandler(self.AWS_ACCESS_KEY_ID, \
+        s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \
                 self.AWS_SECRET_ACCESS_KEY, \
                 httpdownloadhandler=HttpDownloadHandlerMock)
         self.download_request = s3reqh.download_request