Refactored Scrapyd code to fix a couple of bugs that ocurred when running projects without eggs

scrapyd/poller.py

@@ -9,8 +9,7 @@ class QueuePoller(object):
     implements(IPoller)
 
     def __init__(self, config):
-        self.eggs_dir = config.get('eggs_dir', 'eggs')
-        self.dbs_dir = config.get('dbs_dir', 'dbs')
+        self.config = config
         self.update_projects()
         self.dq = DeferredQueue(size=1)
 
@@ -25,7 +24,7 @@ class QueuePoller(object):
         return self.dq.get()
 
     def update_projects(self):
-        self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir)
+        self.queues = get_spider_queues(self.config)
 
     def _message(self, project):
         return {'project': str(project)}

scrapyd/scheduler.py

@@ -8,8 +8,7 @@ class SpiderScheduler(object):
     implements(ISpiderScheduler)
 
     def __init__(self, config):
-        self.eggs_dir = config.get('eggs_dir', 'eggs')
-        self.dbs_dir = config.get('dbs_dir', 'dbs')
+        self.config = config
         self.update_projects()
 
     def schedule(self, project, spider_name, **spider_args):
@@ -20,4 +19,4 @@ class SpiderScheduler(object):
         return self.queues.keys()
 
     def update_projects(self):
-        self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir)
+        self.queues = get_spider_queues(self.config)

scrapyd/tests/test_poller.py

@@ -21,7 +21,7 @@ class QueuePollerTest(unittest.TestCase):
         os.makedirs(os.path.join(eggs_dir, 'mybot1'))
         os.makedirs(os.path.join(eggs_dir, 'mybot2'))
         config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
-        self.queues = get_spider_queues(eggs_dir, dbs_dir)
+        self.queues = get_spider_queues(config)
         self.poller = QueuePoller(config)
 
     def test_interface(self):

scrapyd/tests/test_scheduler.py

@@ -21,7 +21,7 @@ class SpiderSchedulerTest(unittest.TestCase):
         os.makedirs(os.path.join(eggs_dir, 'mybot1'))
         os.makedirs(os.path.join(eggs_dir, 'mybot2'))
         config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
-        self.queues = get_spider_queues(eggs_dir, dbs_dir)
+        self.queues = get_spider_queues(config)
         self.sched = SpiderScheduler(config)
 
     def test_interface(self):

scrapyd/utils.py

@@ -1,12 +1,30 @@
 import os
-import pkg_resources
+from ConfigParser import NoSectionError
 
 from scrapy.spiderqueue import SqliteSpiderQueue
 
-def get_spider_queues(eggsdir, dbsdir):
+def get_spider_queues(config):
     """Return a dict of Spider Quees keyed by project name"""
+    dbsdir = config.get('dbs_dir', 'dbs')
+    if not os.path.exists(dbsdir):
+        os.makedirs(dbsdir)
     d = {}
-    for project in os.listdir(eggsdir):
+    for project in get_project_list(config):
         dbpath = os.path.join(dbsdir, '%s.db' % project)
         d[project] = SqliteSpiderQueue(dbpath)
     return d
+
+def get_project_list(config):
+    """Get list of projects by inspecting the eggs dir and the ones defined in
+    the scrapyd.conf [settings] section
+    """
+    eggs_dir = config.get('eggs_dir', 'eggs')
+    if os.path.exists(eggs_dir):
+        projects = os.listdir(eggs_dir)
+    else:
+        projects = []
+    try:
+        projects += [x[0] for x in config.cp.items('settings')]
+    except NoSectionError:
+        pass
+    return projects