From c1cd019f56b703509bff5cf97da7e2c70fb466b4 Mon Sep 17 00:00:00 2001 From: Alexander Sibiryakov Date: Tue, 31 Mar 2015 10:56:31 +0200 Subject: [PATCH] Setting maximum thread pool size in reactor. --- scrapy/crawler.py | 3 +++ scrapy/settings/default_settings.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 6866be809..41716852c 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -12,6 +12,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object from scrapy import log, signals +from pdb import set_trace class Crawler(object): @@ -152,6 +153,8 @@ class CrawlerProcess(CrawlerRunner): if self.settings.getbool('DNSCACHE_ENABLED'): reactor.installResolver(CachingThreadedResolver(reactor)) + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 0342b1ada..5b8dc4eaa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -194,6 +194,8 @@ NEWSPIDER_MODULE = '' RANDOMIZE_DOWNLOAD_DELAY = True +REACTOR_THREADPOOL_MAXSIZE = 10 + REDIRECT_ENABLED = True REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2