From c1cd019f56b703509bff5cf97da7e2c70fb466b4 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Tue, 31 Mar 2015 10:56:31 +0200 Subject: [PATCH 1/6] Setting maximum thread pool size in reactor. --- scrapy/crawler.py | 3 +++ scrapy/settings/default_settings.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6866be809..41716852c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -12,6 +12,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals +from pdb import set_trace class Crawler(object): @@ -152,6 +153,8 @@ class CrawlerProcess(CrawlerRunner): if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0342b1ada..5b8dc4eaa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -194,6 +194,8 @@ NEWSPIDER_MODULE = '' RANDOMIZE_DOWNLOAD_DELAY = True +REACTOR_THREADPOOL_MAXSIZE = 10 + REDIRECT_ENABLED = True REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 From 5864d291d4ff96351661ce01c414f2bdae43efe0 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Tue, 31 Mar 2015 11:10:56 +0200 Subject: [PATCH 2/6] Setting documentation. --- docs/topics/settings.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 5e11e473f..4812de8b9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -742,6 +742,17 @@ If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. .. _wget: http://www.gnu.org/software/wget/manual/wget.html +.. setting:: REACTOR_THREADPOOL_MAXSIZE + +REACTOR_THREADPOOL_MAXSIZE +------------------ + +Default: ``10`` + +The maximum limit for Twisted Reactor thread pool size. This is common multi-purpose thread pool used by various +Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if +you're experiencing problems with insufficient blocking IO. + .. setting:: REDIRECT_MAX_TIMES REDIRECT_MAX_TIMES From 5916df64cd20ccc5b7fe02410b7ea485892a39da Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 1 Apr 2015 11:24:12 +0200 Subject: [PATCH 3/6] Removing unnecessary import. --- scrapy/crawler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 41716852c..52d4069ac 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -12,7 +12,6 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals -from pdb import set_trace class Crawler(object): From 94fceb4c1500a7add5b462beab83ea0562479880 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 1 Apr 2015 11:25:10 +0200 Subject: [PATCH 4/6] Fixing underscore size. --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4812de8b9..bbe822046 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -745,7 +745,7 @@ If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect. .. setting:: REACTOR_THREADPOOL_MAXSIZE REACTOR_THREADPOOL_MAXSIZE ------------------- +-------------------------- Default: ``10`` From e7b274edf3aa8320edf91a652e1c221654e26aa6 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 1 Apr 2015 11:49:55 +0200 Subject: [PATCH 5/6] Reformat to 80 characters per line. --- docs/topics/settings.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index bbe822046..3a2876a7a 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -749,9 +749,10 @@ REACTOR_THREADPOOL_MAXSIZE Default: ``10`` -The maximum limit for Twisted Reactor thread pool size. This is common multi-purpose thread pool used by various -Scrapy components. Threaded DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase this value if -you're experiencing problems with insufficient blocking IO. +The maximum limit for Twisted Reactor thread pool size. This is common +multi-purpose thread pool used by various Scrapy components. Threaded +DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase +this value if you're experiencing problems with insufficient blocking IO. .. setting:: REDIRECT_MAX_TIMES From b794cdaf4bf5d43d5eb5d3802eb46976d5d55d14 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Wed, 1 Apr 2015 12:07:03 +0200 Subject: [PATCH 6/6] Broad crawls notes. --- docs/topics/broad-crawls.rst | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index b95974f5d..aaf46bc92 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -57,6 +57,27 @@ To increase the global concurrency use:: CONCURRENT_REQUESTS = 100 +Increase Twisted IO thread pool maximum size +============================================ + +Currently Scrapy does DNS resolution in a blocking way with usage of thread +pool. With higher concurrency levels the crawling could be slow or even fail +hitting DNS resolver timeouts. Possible solution to increase the number of +threads handling DNS queries. The DNS queue will be processed faster speeding +up establishing of connection and crawling overall. + +To increase maximum thread pool size use:: + + REACTOR_THREADPOOL_MAXSIZE = 20 + +Setup your own DNS +================== + +If you have multiple crawling processes and single central DNS, it can act +like DoS attack on the DNS server resulting to slow down of entire network or +even blocking your machines. To avoid this setup your own DNS server with +local cache and upstream to some large DNS like OpenDNS or Verizon. + Reduce log level ================