adjust http11 pool size to per-domain concurrency

scrapy/core/downloader/handlers/http11.py

@@ -20,11 +20,11 @@ from scrapy.utils.misc import load_object
 from scrapy import log
 
 
-
 class HTTP11DownloadHandler(object):
 
     def __init__(self, settings):
         self._pool = HTTPConnectionPool(reactor, persistent=True)
+        self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
         self._contextFactory = self._contextFactoryClass()