From 65fc2fbd1f0c84ee56c53c0862ea2af625f7b3e9 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 4 Feb 2011 13:30:01 -0200 Subject: [PATCH] Set CONCURRENT_SPIDERS=1 in Scrapyd to force one spider per process --- scrapyd/environ.py | 1 + scrapyd/tests/test_environ.py | 1 + 2 files changed, 2 insertions(+) diff --git a/scrapyd/environ.py b/scrapyd/environ.py index 8b208d76e..1be3db39a 100644 --- a/scrapyd/environ.py +++ b/scrapyd/environ.py @@ -30,6 +30,7 @@ class Environment(object): dbpath = os.path.join(self.dbs_dir, '%s.db' % project) env['SCRAPY_SQLITE_DB'] = dbpath env['SCRAPY_LOG_FILE'] = self._get_log_file(message) + env['SCRAPY_CONCURRENT_SPIDERS'] = '1' # scrapyd runs one spider per process return env def _get_log_file(self, message): diff --git a/scrapyd/tests/test_environ.py b/scrapyd/tests/test_environ.py index e8230a3e9..30605756c 100644 --- a/scrapyd/tests/test_environ.py +++ b/scrapyd/tests/test_environ.py @@ -29,6 +29,7 @@ class EnvironmentTest(unittest.TestCase): self.assertEqual(env['SCRAPY_SLOT'], '3') self.assertEqual(env['SCRAPY_SPIDER'], 'myspider') self.assertEqual(env['SCRAPY_JOB'], 'ID') + self.assertEqual(env['SCRAPY_CONCURRENT_SPIDERS'], '1') self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) self.assert_(env['SCRAPY_LOG_FILE'].endswith('/mybot/myspider/ID.log')) self.failIf('SCRAPY_SETTINGS_MODULE' in env)