diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f9d0b303c..bd9dcc1ad 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -276,15 +276,6 @@ performed to any single IP. If non-zero, the used instead. In other words, concurrency limits will be applied per IP, not per domain. -.. setting:: CONCURRENT_SPIDERS - -CONCURRENT_SPIDERS ------------------- - -Default: ``8`` - -Maximum number of spiders to scrape in parallel. - .. setting:: DEFAULT_ITEM_CLASS DEFAULT_ITEM_CLASS diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index e545e89c2..c56cd26ab 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. For more information see docs/topics/architecture.rst """ +import warnings from time import time from twisted.internet import defer @@ -13,7 +14,7 @@ from scrapy import log, signals from scrapy.stats import stats from scrapy.core.downloader import Downloader from scrapy.core.scraper import Scraper -from scrapy.exceptions import DontCloseSpider +from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning from scrapy.http import Response, Request from scrapy.utils.misc import load_object from scrapy.utils.signal import send_catch_log, send_catch_log_deferred @@ -59,7 +60,10 @@ class ExecutionEngine(object): self.scheduler_cls = load_object(self.settings['SCHEDULER']) self.downloader = Downloader(crawler) self.scraper = Scraper(crawler) - self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS') + self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS', 1) + if self._concurrent_spiders != 1: + warnings.warn("CONCURRENT_SPIDERS settings is deprecated, use " \ + "Scrapyd max_proc config instead", ScrapyDeprecationWarning) self._spider_closed_callback = spider_closed_callback @defer.inlineCallbacks diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2cb4ebcad..8dfca7215 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -31,8 +31,6 @@ CONCURRENT_REQUESTS = 16 CONCURRENT_REQUESTS_PER_DOMAIN = 8 CONCURRENT_REQUESTS_PER_IP = 0 -CONCURRENT_SPIDERS = 8 - COOKIES_ENABLED = True COOKIES_DEBUG = False diff --git a/scrapyd/environ.py b/scrapyd/environ.py index 1be3db39a..8b208d76e 100644 --- a/scrapyd/environ.py +++ b/scrapyd/environ.py @@ -30,7 +30,6 @@ class Environment(object): dbpath = os.path.join(self.dbs_dir, '%s.db' % project) env['SCRAPY_SQLITE_DB'] = dbpath env['SCRAPY_LOG_FILE'] = self._get_log_file(message) - env['SCRAPY_CONCURRENT_SPIDERS'] = '1' # scrapyd runs one spider per process return env def _get_log_file(self, message): diff --git a/scrapyd/tests/test_environ.py b/scrapyd/tests/test_environ.py index f33363713..220145fd2 100644 --- a/scrapyd/tests/test_environ.py +++ b/scrapyd/tests/test_environ.py @@ -29,7 +29,6 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(env['SCRAPY_SLOT'], '3') self.assertEqual(env['SCRAPY_SPIDER'], 'myspider') self.assertEqual(env['SCRAPY_JOB'], 'ID') - self.assertEqual(env['SCRAPY_CONCURRENT_SPIDERS'], '1') self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log'))) self.failIf('SCRAPY_SETTINGS_MODULE' in env)