fixed some compatibility issues with python 2.5 in scrapyd

scrapy/utils/py26.py

@@ -6,6 +6,7 @@ in Python 2.5. The Python 2.6 function is used when available.
 import sys
 import os
 import fnmatch
+import pkgutil
 from shutil import copy2, copystat
 
 __all__ = ['cpu_count', 'copytree', 'ignore_patterns']
@@ -105,3 +106,27 @@ except ImportError:
         import simplejson as json
     except ImportError:
         import scrapy.xlib.simplejson as json
+
+
+def _get_data(package, resource):
+    loader = pkgutil.get_loader(package)
+    if loader is None or not hasattr(loader, 'get_data'):
+        return None
+    mod = sys.modules.get(package) or loader.load_module(package)
+    if mod is None or not hasattr(mod, '__file__'):
+        return None
+
+    # Modify the resource name to be compatible with the loader.get_data
+    # signature - an os.path format "filename" starting with the dirname of
+    # the package's __file__
+    parts = resource.split('/')
+    parts.insert(0, os.path.dirname(mod.__file__))
+    resource_name = os.path.join(*parts)
+    return loader.get_data(resource_name)
+
+# pkgutil.get_data() not available in python 2.5
+# see http://docs.python.org/release/2.5/lib/module-pkgutil.html
+try:
+    get_data = pkgutil.get_data
+except AttributeError:
+    get_data = _get_data

scrapyd/config.py

@@ -1,8 +1,11 @@
 import glob
-import pkgutil
 from cStringIO import StringIO
 from ConfigParser import SafeConfigParser, NoSectionError, NoOptionError
 
+from scrapy.utils.py26 import get_data
+
+__package__ = 'scrapyd' # required for compatibility with python 2.5
+
 class Config(object):
     """A ConfigParser wrapper to support defaults when calling instance
     methods, and also tied to a single section"""
@@ -12,7 +15,7 @@ class Config(object):
     def __init__(self, values=None):
         if values is None:
             sources = self._getsources()
-            default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf')
+            default_config = get_data(__package__, 'default_scrapyd.conf')
             self.cp = SafeConfigParser()
             self.cp.readfp(StringIO(default_config))
             self.cp.read(sources)

scrapyd/tests/test_eggutils.py

@@ -1,11 +1,14 @@
-import pkgutil, unittest
+import unittest
 from cStringIO import StringIO
 
 from scrapyd.eggutils import get_spider_list_from_eggfile
+from scrapy.utils.py26 import get_data
+
+__package__ = 'scrapyd.tests' # required for compatibility with python 2.5
 
 class EggUtilsTest(unittest.TestCase):
 
     def test_get_spider_list_from_eggfile(self):
-        eggfile = StringIO(pkgutil.get_data(__package__, 'mybot.egg'))
+        eggfile = StringIO(get_data(__package__, 'mybot.egg'))
         spiders = get_spider_list_from_eggfile(eggfile, 'mybot')
         self.assertEqual(set(spiders), set(['spider1', 'spider2']))