diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index b95974f5d..aaf46bc92 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -57,6 +57,27 @@ To increase the global concurrency use:: CONCURRENT_REQUESTS = 100 +Increase Twisted IO thread pool maximum size +============================================ + +Currently Scrapy does DNS resolution in a blocking way with usage of thread +pool. With higher concurrency levels the crawling could be slow or even fail +hitting DNS resolver timeouts. Possible solution to increase the number of +threads handling DNS queries. The DNS queue will be processed faster speeding +up establishing of connection and crawling overall. + +To increase maximum thread pool size use:: + + REACTOR_THREADPOOL_MAXSIZE = 20 + +Setup your own DNS +================== + +If you have multiple crawling processes and single central DNS, it can act +like DoS attack on the DNS server resulting to slow down of entire network or +even blocking your machines. To avoid this setup your own DNS server with +local cache and upstream to some large DNS like OpenDNS or Verizon. + Reduce log level ================ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0f41c427b..c39601b14 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -744,6 +744,18 @@ If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. .. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. setting:: REACTOR_THREADPOOL_MAXSIZE + +REACTOR_THREADPOOL_MAXSIZE +-------------------------- + +Default: ``10`` + +The maximum limit for Twisted Reactor thread pool size. This is common +multi-purpose thread pool used by various Scrapy components. Threaded +DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase +this value if you're experiencing problems with insufficient blocking IO. + .. setting:: REDIRECT_MAX_TIMES REDIRECT_MAX_TIMES diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6866be809..52d4069ac 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -152,6 +152,8 @@ class CrawlerProcess(CrawlerRunner): if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0342b1ada..5b8dc4eaa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -194,6 +194,8 @@ NEWSPIDER_MODULE = '' RANDOMIZE_DOWNLOAD_DELAY = True +REACTOR_THREADPOOL_MAXSIZE = 10 + REDIRECT_ENABLED = True REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2