Merge pull request #1123 from sibiryakov/reactor-threadpool-size

[MRG+1] Reactor threadpool max size setting

docs/topics/broad-crawls.rst

@@ -57,6 +57,27 @@ To increase the global concurrency use::
 
     CONCURRENT_REQUESTS = 100
 
+Increase Twisted IO thread pool maximum size
+============================================
+
+Currently Scrapy does DNS resolution in a blocking way with usage of thread
+pool. With higher concurrency levels the crawling could be slow or even fail
+hitting DNS resolver timeouts. Possible solution to increase the number of
+threads handling DNS queries. The DNS queue will be processed faster speeding
+up establishing of connection and crawling overall.
+
+To increase maximum thread pool size use::
+
+    REACTOR_THREADPOOL_MAXSIZE = 20
+
+Setup your own DNS
+==================
+
+If you have multiple crawling processes and single central DNS, it can act
+like DoS attack on the DNS server resulting to slow down of entire network or
+even blocking your machines. To avoid this setup your own DNS server with
+local cache and upstream to some large DNS like OpenDNS or Verizon.
+
 Reduce log level
 ================
 

docs/topics/settings.rst

@@ -744,6 +744,18 @@ If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
 
 .. _wget: http://www.gnu.org/software/wget/manual/wget.html
 
+.. setting:: REACTOR_THREADPOOL_MAXSIZE
+
+REACTOR_THREADPOOL_MAXSIZE
+--------------------------
+
+Default: ``10``
+
+The maximum limit for Twisted Reactor thread pool size. This is common
+multi-purpose thread pool used by various Scrapy components. Threaded
+DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
+this value if you're experiencing problems with insufficient blocking IO.
+
 .. setting:: REDIRECT_MAX_TIMES
 
 REDIRECT_MAX_TIMES

scrapy/crawler.py

@@ -152,6 +152,8 @@ class CrawlerProcess(CrawlerRunner):
         if self.settings.getbool('DNSCACHE_ENABLED'):
             reactor.installResolver(CachingThreadedResolver(reactor))
 
+        tp = reactor.getThreadPool()
+        tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 

scrapy/settings/default_settings.py

@@ -194,6 +194,8 @@ NEWSPIDER_MODULE = ''
 
 RANDOMIZE_DOWNLOAD_DELAY = True
 
+REACTOR_THREADPOOL_MAXSIZE = 10
+
 REDIRECT_ENABLED = True
 REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2