From f3769651af2a375572f72025dfdeaf6d6cf1f8ce Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 22 Sep 2010 01:04:15 -0300
Subject: [PATCH] Refactored Scrapyd code to fix a couple of bugs that ocurred
 when running projects without eggs

---
 scrapyd/poller.py               |  5 ++---
 scrapyd/scheduler.py            |  5 ++---
 scrapyd/tests/test_poller.py    |  2 +-
 scrapyd/tests/test_scheduler.py |  2 +-
 scrapyd/utils.py                | 24 +++++++++++++++++++++---
 5 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/scrapyd/poller.py b/scrapyd/poller.py
index 37a5c6922..7a6ec4668 100644
--- a/scrapyd/poller.py
+++ b/scrapyd/poller.py
@@ -9,8 +9,7 @@ class QueuePoller(object):
     implements(IPoller)
 
     def __init__(self, config):
-        self.eggs_dir = config.get('eggs_dir', 'eggs')
-        self.dbs_dir = config.get('dbs_dir', 'dbs')
+        self.config = config
         self.update_projects()
         self.dq = DeferredQueue(size=1)
 
@@ -25,7 +24,7 @@ class QueuePoller(object):
         return self.dq.get()
 
     def update_projects(self):
-        self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir)
+        self.queues = get_spider_queues(self.config)
 
     def _message(self, project):
         return {'project': str(project)}
diff --git a/scrapyd/scheduler.py b/scrapyd/scheduler.py
index f96380d72..885b8ea2a 100644
--- a/scrapyd/scheduler.py
+++ b/scrapyd/scheduler.py
@@ -8,8 +8,7 @@ class SpiderScheduler(object):
     implements(ISpiderScheduler)
 
     def __init__(self, config):
-        self.eggs_dir = config.get('eggs_dir', 'eggs')
-        self.dbs_dir = config.get('dbs_dir', 'dbs')
+        self.config = config
         self.update_projects()
 
     def schedule(self, project, spider_name, **spider_args):
@@ -20,4 +19,4 @@ class SpiderScheduler(object):
         return self.queues.keys()
 
     def update_projects(self):
-        self.queues = get_spider_queues(self.eggs_dir, self.dbs_dir)
+        self.queues = get_spider_queues(self.config)
diff --git a/scrapyd/tests/test_poller.py b/scrapyd/tests/test_poller.py
index d8dae49ac..36bbf04a5 100644
--- a/scrapyd/tests/test_poller.py
+++ b/scrapyd/tests/test_poller.py
@@ -21,7 +21,7 @@ class QueuePollerTest(unittest.TestCase):
         os.makedirs(os.path.join(eggs_dir, 'mybot1'))
         os.makedirs(os.path.join(eggs_dir, 'mybot2'))
         config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
-        self.queues = get_spider_queues(eggs_dir, dbs_dir)
+        self.queues = get_spider_queues(config)
         self.poller = QueuePoller(config)
 
     def test_interface(self):
diff --git a/scrapyd/tests/test_scheduler.py b/scrapyd/tests/test_scheduler.py
index a9fb7e6f0..ce655966d 100644
--- a/scrapyd/tests/test_scheduler.py
+++ b/scrapyd/tests/test_scheduler.py
@@ -21,7 +21,7 @@ class SpiderSchedulerTest(unittest.TestCase):
         os.makedirs(os.path.join(eggs_dir, 'mybot1'))
         os.makedirs(os.path.join(eggs_dir, 'mybot2'))
         config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
-        self.queues = get_spider_queues(eggs_dir, dbs_dir)
+        self.queues = get_spider_queues(config)
         self.sched = SpiderScheduler(config)
 
     def test_interface(self):
diff --git a/scrapyd/utils.py b/scrapyd/utils.py
index c8b29df31..1ab368aa5 100644
--- a/scrapyd/utils.py
+++ b/scrapyd/utils.py
@@ -1,12 +1,30 @@
 import os
-import pkg_resources
+from ConfigParser import NoSectionError
 
 from scrapy.spiderqueue import SqliteSpiderQueue
 
-def get_spider_queues(eggsdir, dbsdir):
+def get_spider_queues(config):
     """Return a dict of Spider Quees keyed by project name"""
+    dbsdir = config.get('dbs_dir', 'dbs')
+    if not os.path.exists(dbsdir):
+        os.makedirs(dbsdir)
     d = {}
-    for project in os.listdir(eggsdir):
+    for project in get_project_list(config):
         dbpath = os.path.join(dbsdir, '%s.db' % project)
         d[project] = SqliteSpiderQueue(dbpath)
     return d
+
+def get_project_list(config):
+    """Get list of projects by inspecting the eggs dir and the ones defined in
+    the scrapyd.conf [settings] section
+    """
+    eggs_dir = config.get('eggs_dir', 'eggs')
+    if os.path.exists(eggs_dir):
+        projects = os.listdir(eggs_dir)
+    else:
+        projects = []
+    try:
+        projects += [x[0] for x in config.cp.items('settings')]
+    except NoSectionError:
+        pass
+    return projects