Deprecate unused squeues (#5117)

scrapy/squeues.py

@@ -8,6 +8,7 @@ import pickle
 
 from queuelib import queue
 
+from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.reqser import request_to_dict, request_from_dict
 
 
@@ -123,30 +124,60 @@ def _pickle_serialize(obj):
         raise ValueError(str(e)) from e
 
 
-PickleFifoDiskQueueNonRequest = _serializable_queue(
+_PickleFifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.FifoDiskQueue),
     _pickle_serialize,
     pickle.loads
 )
-PickleLifoDiskQueueNonRequest = _serializable_queue(
+_PickleLifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.LifoDiskQueue),
     _pickle_serialize,
     pickle.loads
 )
-MarshalFifoDiskQueueNonRequest = _serializable_queue(
+_MarshalFifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.FifoDiskQueue),
     marshal.dumps,
     marshal.loads
 )
-MarshalLifoDiskQueueNonRequest = _serializable_queue(
+_MarshalLifoSerializationDiskQueue = _serializable_queue(
     _with_mkdir(queue.LifoDiskQueue),
     marshal.dumps,
     marshal.loads
 )
 
-PickleFifoDiskQueue = _scrapy_serialization_queue(PickleFifoDiskQueueNonRequest)
-PickleLifoDiskQueue = _scrapy_serialization_queue(PickleLifoDiskQueueNonRequest)
-MarshalFifoDiskQueue = _scrapy_serialization_queue(MarshalFifoDiskQueueNonRequest)
-MarshalLifoDiskQueue = _scrapy_serialization_queue(MarshalLifoDiskQueueNonRequest)
+# public queue classes
+PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQueue)
+PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
+MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
+MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
 FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
 LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
+
+
+# deprecated queue classes
+_subclass_warn_message = "{cls} inherits from deprecated class {old}"
+_instance_warn_message = "{cls} is deprecated"
+PickleFifoDiskQueueNonRequest = create_deprecated_class(
+    name="PickleFifoDiskQueueNonRequest",
+    new_class=_PickleFifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
+PickleLifoDiskQueueNonRequest = create_deprecated_class(
+    name="PickleLifoDiskQueueNonRequest",
+    new_class=_PickleLifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
+MarshalFifoDiskQueueNonRequest = create_deprecated_class(
+    name="MarshalFifoDiskQueueNonRequest",
+    new_class=_MarshalFifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)
+MarshalLifoDiskQueueNonRequest = create_deprecated_class(
+    name="MarshalLifoDiskQueueNonRequest",
+    new_class=_MarshalLifoSerializationDiskQueue,
+    subclass_warn_message=_subclass_warn_message,
+    instance_warn_message=_instance_warn_message,
+)

tests/test_squeues.py

@@ -3,10 +3,10 @@ import sys
 
 from queuelib.tests import test_queue as t
 from scrapy.squeues import (
-    MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
-    MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue,
-    PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue,
-    PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue
+    _MarshalFifoSerializationDiskQueue,
+    _MarshalLifoSerializationDiskQueue,
+    _PickleFifoSerializationDiskQueue,
+    _PickleLifoSerializationDiskQueue,
 )
 from scrapy.item import Item, Field
 from scrapy.http import Request
@@ -53,7 +53,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
     chunksize = 100000
 
     def queue(self):
-        return MarshalFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+        return _MarshalFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
 
 
 class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
@@ -77,7 +77,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
     chunksize = 100000
 
     def queue(self):
-        return PickleFifoDiskQueue(self.qpath, chunksize=self.chunksize)
+        return _PickleFifoSerializationDiskQueue(self.qpath, chunksize=self.chunksize)
 
     def test_serialize_item(self):
         q = self.queue()
@@ -155,13 +155,13 @@ class LifoDiskQueueTestMixin:
 class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
-        return MarshalLifoDiskQueue(self.qpath)
+        return _MarshalLifoSerializationDiskQueue(self.qpath)
 
 
 class PickleLifoDiskQueueTest(t.LifoDiskQueueTest, LifoDiskQueueTestMixin):
 
     def queue(self):
-        return PickleLifoDiskQueue(self.qpath)
+        return _PickleLifoSerializationDiskQueue(self.qpath)
 
     def test_serialize_item(self):
         q = self.queue()