RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Set CONCURRENT_SPIDERS=1 in Scrapyd to force one spider per process

This commit is contained in:
Pablo Hoffman 2011-02-04 13:30:01 -02:00
parent b1c89508f5
commit 65fc2fbd1f
2 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapyd/environ.py
View File

@ -30,6 +30,7 @@ class Environment(object):
dbpath = os.path.join(self.dbs_dir, '%s.db' % project)
env['SCRAPY_SQLITE_DB'] = dbpath
env['SCRAPY_LOG_FILE'] = self._get_log_file(message)
env['SCRAPY_CONCURRENT_SPIDERS'] = '1' # scrapyd runs one spider per process
return env
def _get_log_file(self, message):

1
scrapyd/tests/test_environ.py
View File

@ -29,6 +29,7 @@ class EnvironmentTest(unittest.TestCase):
self.assertEqual(env['SCRAPY_SLOT'], '3')
self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
self.assertEqual(env['SCRAPY_JOB'], 'ID')
self.assertEqual(env['SCRAPY_CONCURRENT_SPIDERS'], '1')
self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
self.assert_(env['SCRAPY_LOG_FILE'].endswith('/mybot/myspider/ID.log'))
self.failIf('SCRAPY_SETTINGS_MODULE' in env)