RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate unused squeues (#5117)

This commit is contained in:
Eugenio Lacuesta 2021-04-27 09:41:44 -03:00 committed by GitHub
parent ddea6b7bfa
commit 02ae1deaf4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
scrapy/squeues.py
View File

@ -8,6 +8,7 @@ import pickle
from queuelib import queue from queuelib import queue
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.reqser import request_to_dict, request_from_dict from scrapy.utils.reqser import request_to_dict, request_from_dict
@ -123,30 +124,60 @@ def _pickle_serialize(obj):
raise ValueError(str(e)) from e raise ValueError(str(e)) from e
PickleFifoDiskQueueNonRequest = _serializable_queue( _PickleFifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.FifoDiskQueue), _with_mkdir(queue.FifoDiskQueue),
_pickle_serialize, _pickle_serialize,
pickle.loads pickle.loads
) )
PickleLifoDiskQueueNonRequest = _serializable_queue( _PickleLifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.LifoDiskQueue), _with_mkdir(queue.LifoDiskQueue),
_pickle_serialize, _pickle_serialize,
pickle.loads pickle.loads
) )
MarshalFifoDiskQueueNonRequest = _serializable_queue( _MarshalFifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.FifoDiskQueue), _with_mkdir(queue.FifoDiskQueue),
marshal.dumps, marshal.dumps,
marshal.loads marshal.loads
) )
MarshalLifoDiskQueueNonRequest = _serializable_queue( _MarshalLifoSerializationDiskQueue = _serializable_queue(
_with_mkdir(queue.LifoDiskQueue), _with_mkdir(queue.LifoDiskQueue),
marshal.dumps, marshal.dumps,
marshal.loads marshal.loads
) )
PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest) # public queue classes
PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest) PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)
MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest) PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest) MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue) FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue) LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
# deprecated queue classes
_subclass_warn_message = "{cls} inherits from deprecated class {old}"
_instance_warn_message = "{cls} is deprecated"
PickleFifoDiskQueueNonRequest = create_deprecated_class(
name="PickleFifoDiskQueueNonRequest",
new_class=_PickleFifoSerializationDiskQueue,
subclass_warn_message=_subclass_warn_message,
instance_warn_message=_instance_warn_message,
)
PickleLifoDiskQueueNonRequest = create_deprecated_class(
name="PickleLifoDiskQueueNonRequest",
new_class=_PickleLifoSerializationDiskQueue,
subclass_warn_message=_subclass_warn_message,
instance_warn_message=_instance_warn_message,
)
MarshalFifoDiskQueueNonRequest = create_deprecated_class(
name="MarshalFifoDiskQueueNonRequest",
new_class=_MarshalFifoSerializationDiskQueue,
subclass_warn_message=_subclass_warn_message,
instance_warn_message=_instance_warn_message,
)
MarshalLifoDiskQueueNonRequest = create_deprecated_class(
name="MarshalLifoDiskQueueNonRequest",
new_class=_MarshalLifoSerializationDiskQueue,
subclass_warn_message=_subclass_warn_message,
instance_warn_message=_instance_warn_message,
)

16
tests/test_squeues.py
View File

@ -3,10 +3,10 @@ import sys
from queuelib.tests import test_queue as t from queuelib.tests import test_queue as t
from scrapy.squeues import ( from scrapy.squeues import (
MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue, _MarshalFifoSerializationDiskQueue,
MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue, _MarshalLifoSerializationDiskQueue,
PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue, _PickleFifoSerializationDiskQueue,
PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue _PickleLifoSerializationDiskQueue,
) )
from scrapy.item import Item, Field from scrapy.item import Item, Field
from scrapy.http import Request from scrapy.http import Request
@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
chunksize = 100000 chunksize = 100000
def queue(self): def queue(self):
return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize) return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
chunksize = 100000 chunksize = 100000
def queue(self): def queue(self):
return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize) return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
def test_serialize_item(self): def test_serialize_item(self):
q = self.queue() q = self.queue()
@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin:
class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def queue(self): def queue(self):
return MarshalLifoDiskQueue(self.qpath) return _MarshalLifoSerializationDiskQueue(self.qpath)
class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin): class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
def queue(self): def queue(self):
return PickleLifoDiskQueue(self.qpath) return _PickleLifoSerializationDiskQueue(self.qpath)
def test_serialize_item(self): def test_serialize_item(self):
q = self.queue() q = self.queue()