persistent scheduler: use pickle (instead of marshal) as the default serialization format, to support serializing more objects out of the box. also removed __slots__ from Request/Response objects to make them serializable by default.

scrapy/http/headers.py

@@ -5,8 +5,6 @@ from scrapy.utils.datatypes import CaselessDict
 class Headers(CaselessDict):
     """Case insensitive http headers dictionary"""
 
-    __slots__ = ['encoding']
-
     def __init__(self, seq=None, encoding='utf-8'):
         self.encoding = encoding
         super(Headers, self).__init__(seq)

scrapy/http/request/__init__.py

@@ -16,10 +16,6 @@ from scrapy.http.common import deprecated_setter
 
 class Request(object_ref):
 
-    __slots__ = ['_encoding', 'method', '_url', '_body', '_meta', \
-        'dont_filter', 'headers', 'cookies', 'callback', 'errback', 'priority', \
-        '__weakref__']
-
     def __init__(self, url, callback=None, method='GET', headers=None, body=None, 
                  cookies=None, meta=None, encoding='utf-8', priority=0,
                  dont_filter=False, errback=None):

scrapy/http/request/form.py

@@ -22,8 +22,6 @@ def _unicode_to_str(string, encoding):
 
 class FormRequest(Request):
 
-    __slots__ = ()
-
     def __init__(self, *args, **kwargs):
         formdata = kwargs.pop('formdata', None)
         super(FormRequest, self).__init__(*args, **kwargs)

scrapy/http/request/rpc.py

@@ -15,8 +15,6 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 class XmlRpcRequest(Request):
 
-    __slots__ = ()
-
     def __init__(self, *args, **kwargs):
         encoding = kwargs.get('encoding', None)
         if 'body' not in kwargs and 'params' in kwargs:

scrapy/http/response/__init__.py

@@ -13,9 +13,6 @@ from scrapy.http.common import deprecated_setter
 
 class Response(object_ref):
 
-    __slots__ = ['_url', 'headers', 'status', '_body', 'request', \
-        'flags', '__weakref__']
-
     def __init__(self, url, status=200, headers=None, body='', flags=None, request=None):
         self.headers = Headers(headers or {})
         self.status = int(status)

scrapy/http/response/html.py

@@ -12,8 +12,6 @@ from scrapy.utils.python import memoizemethod_noargs
 
 class HtmlResponse(TextResponse):
 
-    __slots__ = ()
-
     _template = r'''%s\s*=\s*["']?\s*%s\s*["']?'''
 
     _httpequiv_re = _template % ('http-equiv', 'Content-Type')

scrapy/http/response/text.py

@@ -24,8 +24,6 @@ class TextResponse(Response):
     _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING']
     _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I)
 
-    __slots__ = ['_encoding', '_cached_benc', '_cached_ubody']
-
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)
         self._cached_benc = None

scrapy/http/response/xml.py

@@ -12,8 +12,6 @@ from scrapy.utils.python import memoizemethod_noargs
 
 class XmlResponse(TextResponse):
 
-    __slots__ = ()
-
     _template = r'''%s\s*=\s*["']?\s*%s\s*["']?'''
     _encoding_re  = _template % ('encoding', r'(?P<charset>[\w-]+)')
     XMLDECL_RE  = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I)

scrapy/settings/default_settings.py

@@ -223,7 +223,7 @@ RETRY_PRIORITY_ADJUST = -1
 ROBOTSTXT_OBEY = False
 
 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
-SCHEDULER_DISK_QUEUE = 'scrapy.squeue.MarshalDiskQueue'
+SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleDiskQueue'
 
 SELECTORS_BACKEND = None # possible values: libxml2, lxml
 

scrapy/squeue.py

@@ -2,10 +2,22 @@
 Scheduler disk-based queues
 """
 
-import marshal
+import marshal, cPickle as pickle
 
 from scrapy.utils.queue import DiskQueue
 
+
+class PickleDiskQueue(DiskQueue):
+
+    def push(self, obj):
+        super(PickleDiskQueue, self).push(pickle.dumps(obj))
+
+    def pop(self):
+        s = super(PickleDiskQueue, self).pop()
+        if s:
+            return pickle.loads(s)
+
+
 class MarshalDiskQueue(DiskQueue):
 
     def push(self, obj):

scrapy/tests/test_http_headers.py

@@ -119,9 +119,3 @@ class HeadersTest(unittest.TestCase):
         h1.setlistdefault('header2', ['value2', 'value3'])
         self.assertEqual(h1.getlist('header1'), ['value1'])
         self.assertEqual(h1.getlist('header2'), ['value2', 'value3'])
-
-    def test_slots(self):
-        """Check that classes are using slots and are weak-referenceable"""
-        x = Headers({})
-        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
-            x.__class__.__name__

scrapy/tests/test_http_request.py

@@ -1,5 +1,4 @@
 import cgi
-import weakref
 import unittest
 import xmlrpclib
 from inspect import getargspec
@@ -169,13 +168,6 @@ class RequestTest(unittest.TestCase):
         r = self.request_class("http://www.example.com", method=u"POST")
         assert isinstance(r.method, str)
 
-    def test_weakref_slots(self):
-        """Check that classes are using slots and are weak-referenceable"""
-        x = self.request_class('http://www.example.com')
-        weakref.ref(x)
-        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
-            x.__class__.__name__
-
 
 class FormRequestTest(RequestTest):
 

scrapy/tests/test_http_response.py

@@ -1,5 +1,4 @@
 import unittest
-import weakref
 
 from scrapy.http import Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers
 from scrapy.utils.encoding import resolve_encoding
@@ -92,13 +91,6 @@ class BaseResponseTest(unittest.TestCase):
         self.assertEqual(r4.body, '')
         self.assertEqual(r4.flags, [])
 
-    def test_weakref_slots(self):
-        """Check that classes are using slots and are weak-referenceable"""
-        x = self.response_class('http://www.example.com')
-        weakref.ref(x)
-        assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
-            x.__class__.__name__
-
     def _assert_response_values(self, response, encoding, body):
         if isinstance(body, unicode):
             body_unicode = body

scrapy/tests/test_squeue.py

@@ -1,6 +1,18 @@
 from scrapy.tests import test_utils_queue as t
-from scrapy.squeue import MarshalDiskQueue
+from scrapy.squeue import MarshalDiskQueue, PickleDiskQueue
+from scrapy.item import Item, Field
+from scrapy.http import Request
+from scrapy.contrib.loader import ItemLoader
 
+class TestItem(Item):
+    name = Field()
+
+def test_processor(x):
+    return x + x
+
+class TestLoader(ItemLoader):
+    default_item_class = TestItem
+    name_out = staticmethod(test_processor)
 
 class MarshalDiskQueueTest(t.DiskQueueTest):
 
@@ -29,3 +41,60 @@ class ChunkSize3MarshalDiskQueueTest(MarshalDiskQueueTest):
 
 class ChunkSize4MarshalDiskQueueTest(MarshalDiskQueueTest):
     chunksize = 4
+
+
+class PickleDiskQueueTest(t.DiskQueueTest):
+
+    chunksize = 100000
+
+    def queue(self):
+        return PickleDiskQueue(self.qdir, chunksize=self.chunksize)
+
+    def test_serialize(self):
+        q = self.queue()
+        q.push('a')
+        q.push(123)
+        q.push({'a': 'dict'})
+        self.assertEqual(q.pop(), 'a')
+        self.assertEqual(q.pop(), 123)
+        self.assertEqual(q.pop(), {'a': 'dict'})
+
+    def test_serialize_item(self):
+        q = self.queue()
+        i = TestItem(name='foo')
+        q.push(i)
+        i2 = q.pop()
+        assert isinstance(i2, TestItem)
+        self.assertEqual(i, i2)
+
+    def test_serialize_loader(self):
+        q = self.queue()
+        l = TestLoader()
+        q.push(l)
+        l2 = q.pop()
+        assert isinstance(l2, TestLoader)
+        assert l2.default_item_class is TestItem
+        self.assertEqual(l2.name_out('x'), 'xx')
+
+    def test_serialize_request_recursive(self):
+        q = self.queue()
+        r = Request('http://www.example.com')
+        r.meta['request'] = r
+        q.push(r)
+        r2 = q.pop()
+        assert isinstance(r2, Request)
+        self.assertEqual(r.url, r2.url)
+        assert r2.meta['request'] is r2
+
+class ChunkSize1PickleDiskQueueTest(PickleDiskQueueTest):
+    chunksize = 1
+
+class ChunkSize2PickleDiskQueueTest(PickleDiskQueueTest):
+    chunksize = 2
+
+class ChunkSize3PickleDiskQueueTest(PickleDiskQueueTest):
+    chunksize = 3
+
+class ChunkSize4PickleDiskQueueTest(PickleDiskQueueTest):
+    chunksize = 4
+