removed CONCURRENT_SPIDERS setting (use scrapyd maxproc instead)

docs/topics/settings.rst

@@ -276,15 +276,6 @@ performed to any single IP. If non-zero, the
 used instead. In other words, concurrency limits will be applied per IP, not
 per domain.
 
-.. setting:: CONCURRENT_SPIDERS
-
-CONCURRENT_SPIDERS
-------------------
-
-Default: ``8``
-
-Maximum number of spiders to scrape in parallel.
-
 .. setting:: DEFAULT_ITEM_CLASS
 
 DEFAULT_ITEM_CLASS

scrapy/core/engine.py

@@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders.
 For more information see docs/topics/architecture.rst
 
 """
+import warnings
 from time import time
 
 from twisted.internet import defer
@@ -13,7 +14,7 @@ from scrapy import log, signals
 from scrapy.stats import stats
 from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import DontCloseSpider
+from scrapy.exceptions import DontCloseSpider, ScrapyDeprecationWarning
 from scrapy.http import Response, Request
 from scrapy.utils.misc import load_object
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
@@ -59,7 +60,10 @@ class ExecutionEngine(object):
         self.scheduler_cls = load_object(self.settings['SCHEDULER'])
         self.downloader = Downloader(crawler)
         self.scraper = Scraper(crawler)
-        self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS')
+        self._concurrent_spiders = self.settings.getint('CONCURRENT_SPIDERS', 1)
+        if self._concurrent_spiders != 1:
+            warnings.warn("CONCURRENT_SPIDERS settings is deprecated, use " \
+                "Scrapyd max_proc config instead", ScrapyDeprecationWarning)
         self._spider_closed_callback = spider_closed_callback
 
     @defer.inlineCallbacks

scrapy/settings/default_settings.py

@@ -31,8 +31,6 @@ CONCURRENT_REQUESTS = 16
 CONCURRENT_REQUESTS_PER_DOMAIN = 8
 CONCURRENT_REQUESTS_PER_IP = 0
 
-CONCURRENT_SPIDERS = 8
-
 COOKIES_ENABLED = True
 COOKIES_DEBUG = False
 

scrapyd/environ.py

@@ -30,7 +30,6 @@ class Environment(object):
         dbpath = os.path.join(self.dbs_dir, '%s.db' % project)
         env['SCRAPY_SQLITE_DB'] = dbpath
         env['SCRAPY_LOG_FILE'] = self._get_log_file(message)
-        env['SCRAPY_CONCURRENT_SPIDERS'] = '1' # scrapyd runs one spider per process
         return env
 
     def _get_log_file(self, message):

scrapyd/tests/test_environ.py

@@ -29,7 +29,6 @@ class EnvironmentTest(unittest.TestCase):
         self.assertEqual(env['SCRAPY_SLOT'], '3')
         self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
         self.assertEqual(env['SCRAPY_JOB'], 'ID')
-        self.assertEqual(env['SCRAPY_CONCURRENT_SPIDERS'], '1')
         self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
         self.assert_(env['SCRAPY_LOG_FILE'].endswith(os.path.join('mybot', 'myspider', 'ID.log')))
         self.failIf('SCRAPY_SETTINGS_MODULE' in env)