diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 5efeda29c..cffe7aab8 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -5,8 +5,6 @@ from scrapy.utils.datatypes import CaselessDict class Headers(CaselessDict): """Case insensitive http headers dictionary""" - __slots__ = ['encoding'] - def __init__(self, seq=None, encoding='utf-8'): self.encoding = encoding super(Headers, self).__init__(seq) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 3fa41ccb7..ce1552bb2 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -16,10 +16,6 @@ from scrapy.http.common import deprecated_setter class Request(object_ref): - __slots__ = ['_encoding', 'method', '_url', '_body', '_meta', \ - 'dont_filter', 'headers', 'cookies', 'callback', 'errback', 'priority', \ - '__weakref__'] - def __init__(self, url, callback=None, method='GET', headers=None, body=None, cookies=None, meta=None, encoding='utf-8', priority=0, dont_filter=False, errback=None): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b683c6415..fb9272a54 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -22,8 +22,6 @@ def _unicode_to_str(string, encoding): class FormRequest(Request): - __slots__ = () - def __init__(self, *args, **kwargs): formdata = kwargs.pop('formdata', None) super(FormRequest, self).__init__(*args, **kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 14906996d..0e8207f0f 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -15,8 +15,6 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - __slots__ = () - def __init__(self, *args, **kwargs): encoding = kwargs.get('encoding', None) if 'body' not in kwargs and 'params' in kwargs: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index ed4e3e3b4..46be1e362 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -13,9 +13,6 @@ from scrapy.http.common import deprecated_setter class Response(object_ref): - __slots__ = ['_url', 'headers', 'status', '_body', 'request', \ - 'flags', '__weakref__'] - def __init__(self, url, status=200, headers=None, body='', flags=None, request=None): self.headers = Headers(headers or {}) self.status = int(status) diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 1ef52490c..f4236f426 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -12,8 +12,6 @@ from scrapy.utils.python import memoizemethod_noargs class HtmlResponse(TextResponse): - __slots__ = () - _template = r'''%s\s*=\s*["']?\s*%s\s*["']?''' _httpequiv_re = _template % ('http-equiv', 'Content-Type') diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 11c52236b..f6e1ce224 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -24,8 +24,6 @@ class TextResponse(Response): _DEFAULT_ENCODING = settings['DEFAULT_RESPONSE_ENCODING'] _ENCODING_RE = re.compile(r'charset=([\w-]+)', re.I) - __slots__ = ['_encoding', '_cached_benc', '_cached_ubody'] - def __init__(self, *args, **kwargs): self._encoding = kwargs.pop('encoding', None) self._cached_benc = None diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 90a06f34c..21b6420ba 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -12,8 +12,6 @@ from scrapy.utils.python import memoizemethod_noargs class XmlResponse(TextResponse): - __slots__ = () - _template = r'''%s\s*=\s*["']?\s*%s\s*["']?''' _encoding_re = _template % ('encoding', r'(?P[\w-]+)') XMLDECL_RE = re.compile(r'<\?xml\s.*?%s' % _encoding_re, re.I) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 03ea0d68f..62db9ebf2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -223,7 +223,7 @@ RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' -SCHEDULER_DISK_QUEUE = 'scrapy.squeue.MarshalDiskQueue' +SCHEDULER_DISK_QUEUE = 'scrapy.squeue.PickleDiskQueue' SELECTORS_BACKEND = None # possible values: libxml2, lxml diff --git a/scrapy/squeue.py b/scrapy/squeue.py index 2fd4f05ee..27b6fe7b7 100644 --- a/scrapy/squeue.py +++ b/scrapy/squeue.py @@ -2,10 +2,22 @@ Scheduler disk-based queues """ -import marshal +import marshal, cPickle as pickle from scrapy.utils.queue import DiskQueue + +class PickleDiskQueue(DiskQueue): + + def push(self, obj): + super(PickleDiskQueue, self).push(pickle.dumps(obj)) + + def pop(self): + s = super(PickleDiskQueue, self).pop() + if s: + return pickle.loads(s) + + class MarshalDiskQueue(DiskQueue): def push(self, obj): diff --git a/scrapy/tests/test_http_headers.py b/scrapy/tests/test_http_headers.py index 7433ed830..269a3a339 100644 --- a/scrapy/tests/test_http_headers.py +++ b/scrapy/tests/test_http_headers.py @@ -119,9 +119,3 @@ class HeadersTest(unittest.TestCase): h1.setlistdefault('header2', ['value2', 'value3']) self.assertEqual(h1.getlist('header1'), ['value1']) self.assertEqual(h1.getlist('header2'), ['value2', 'value3']) - - def test_slots(self): - """Check that classes are using slots and are weak-referenceable""" - x = Headers({}) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index f65163433..2a3015488 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,5 +1,4 @@ import cgi -import weakref import unittest import xmlrpclib from inspect import getargspec @@ -169,13 +168,6 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com", method=u"POST") assert isinstance(r.method, str) - def test_weakref_slots(self): - """Check that classes are using slots and are weak-referenceable""" - x = self.request_class('http://www.example.com') - weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ - class FormRequestTest(RequestTest): diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index b1edf4a3a..46314c1b1 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -1,5 +1,4 @@ import unittest -import weakref from scrapy.http import Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers from scrapy.utils.encoding import resolve_encoding @@ -92,13 +91,6 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r4.body, '') self.assertEqual(r4.flags, []) - def test_weakref_slots(self): - """Check that classes are using slots and are weak-referenceable""" - x = self.response_class('http://www.example.com') - weakref.ref(x) - assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \ - x.__class__.__name__ - def _assert_response_values(self, response, encoding, body): if isinstance(body, unicode): body_unicode = body diff --git a/scrapy/tests/test_squeue.py b/scrapy/tests/test_squeue.py index 818888d65..0891ae00d 100644 --- a/scrapy/tests/test_squeue.py +++ b/scrapy/tests/test_squeue.py @@ -1,6 +1,18 @@ from scrapy.tests import test_utils_queue as t -from scrapy.squeue import MarshalDiskQueue +from scrapy.squeue import MarshalDiskQueue, PickleDiskQueue +from scrapy.item import Item, Field +from scrapy.http import Request +from scrapy.contrib.loader import ItemLoader +class TestItem(Item): + name = Field() + +def test_processor(x): + return x + x + +class TestLoader(ItemLoader): + default_item_class = TestItem + name_out = staticmethod(test_processor) class MarshalDiskQueueTest(t.DiskQueueTest): @@ -29,3 +41,60 @@ class ChunkSize3MarshalDiskQueueTest(MarshalDiskQueueTest): class ChunkSize4MarshalDiskQueueTest(MarshalDiskQueueTest): chunksize = 4 + + +class PickleDiskQueueTest(t.DiskQueueTest): + + chunksize = 100000 + + def queue(self): + return PickleDiskQueue(self.qdir, chunksize=self.chunksize) + + def test_serialize(self): + q = self.queue() + q.push('a') + q.push(123) + q.push({'a': 'dict'}) + self.assertEqual(q.pop(), 'a') + self.assertEqual(q.pop(), 123) + self.assertEqual(q.pop(), {'a': 'dict'}) + + def test_serialize_item(self): + q = self.queue() + i = TestItem(name='foo') + q.push(i) + i2 = q.pop() + assert isinstance(i2, TestItem) + self.assertEqual(i, i2) + + def test_serialize_loader(self): + q = self.queue() + l = TestLoader() + q.push(l) + l2 = q.pop() + assert isinstance(l2, TestLoader) + assert l2.default_item_class is TestItem + self.assertEqual(l2.name_out('x'), 'xx') + + def test_serialize_request_recursive(self): + q = self.queue() + r = Request('http://www.example.com') + r.meta['request'] = r + q.push(r) + r2 = q.pop() + assert isinstance(r2, Request) + self.assertEqual(r.url, r2.url) + assert r2.meta['request'] is r2 + +class ChunkSize1PickleDiskQueueTest(PickleDiskQueueTest): + chunksize = 1 + +class ChunkSize2PickleDiskQueueTest(PickleDiskQueueTest): + chunksize = 2 + +class ChunkSize3PickleDiskQueueTest(PickleDiskQueueTest): + chunksize = 3 + +class ChunkSize4PickleDiskQueueTest(PickleDiskQueueTest): + chunksize = 4 +