scrapyd: added support for passing custom settings to schedule.json

scrapyd/tests/test_utils.py

@@ -22,6 +22,12 @@ class UtilsTest(unittest.TestCase):
         self.assertEqual(cargs, ['lala', '-a', 'arg1=val1'])
         assert all(isinstance(x, str) for x in cargs), cargs
 
+    def test_get_crawl_args_with_settings(self):
+        msg = {'_project': 'lolo', '_spider': 'lala', 'arg1': u'val1', 'settings': {'ONE': 'two'}}
+        cargs = get_crawl_args(msg)
+        self.assertEqual(cargs, ['lala', '-a', 'arg1=val1', '--set', 'ONE=two'])
+        assert all(isinstance(x, str) for x in cargs), cargs
+
 class GetSpiderListTest(unittest.TestCase):
 
     def test_get_spider_list(self):

scrapyd/utils.py

@@ -40,9 +40,13 @@ def get_crawl_args(message):
     msg = message.copy()
     args = [unicode_to_str(msg['_spider'])]
     del msg['_project'], msg['_spider']
+    settings = msg.pop('settings', {})
     for k, v in stringify_dict(msg, keys_only=False).items():
         args += ['-a']
         args += ['%s=%s' % (k, v)]
+    for k, v in stringify_dict(settings, keys_only=False).items():
+        args += ['--set']
+        args += ['%s=%s' % (k, v)]
     return args
 
 def get_spider_list(project, runner=None):

scrapyd/webservice.py

@@ -23,9 +23,12 @@ class WsResource(JsonResource):
 class Schedule(WsResource):
 
     def render_POST(self, txrequest):
+        settings = txrequest.args.pop('setting', [])
+        settings = dict(x.split('=', 1) for x in settings)
         args = dict((k, v[0]) for k, v in txrequest.args.items())
         project = args.pop('project')
         spider = args.pop('spider')
+        args['settings'] = settings
         jobid = uuid.uuid1().hex
         args['_job'] = jobid
         self.root.scheduler.schedule(project, spider, **args)