RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Chain pickling exception, test_squeues.py updates

This commit is contained in:
Eugenio Lacuesta 2020-05-06 14:05:27 -03:00
parent b1ddd7bd7b
commit 93436f9d3a
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
2 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/squeues.py
View File

@ -82,11 +82,10 @@ def _scrapy_non_serialization_queue(queue_class):
def _pickle_serialize(obj):
try:
return pickle.dumps(obj, protocol=4)
# Python <= 3.4 raises pickle.PicklingError here while
# 3.5 <= Python < 3.6 raises AttributeError and
# Python >= 3.6 raises TypeError
# Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)
# TypeError is raised from parsel.Selector
except (pickle.PicklingError, AttributeError, TypeError) as e:
raise ValueError(str(e))
raise ValueError(str(e)) from e
PickleFifoDiskQueueNonRequest = _serializable_queue(

28
tests/test_squeues.py
View File

@ -28,20 +28,7 @@ class TestLoader(ItemLoader):
def nonserializable_object_test(self):
q = self.queue()
try:
pickle.dumps(lambda x: x)
except Exception:
# Trigger Twisted bug #7989
import twisted.persisted.styles # NOQA
self.assertRaises(ValueError, q.push, lambda x: x)
else:
# Use a different unpickleable object
class A:
pass
a = A()
a.__reduce__ = a.__reduce_ex__ = None
self.assertRaises(ValueError, q.push, a)
self.assertRaises(ValueError, q.push, lambda x: x)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text='<html><body><p>some text</p></body></html>')
self.assertRaises(ValueError, q.push, sel)
@ -118,6 +105,19 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoDiskQueueTestMixin):
self.assertEqual(r.url, r2.url)
assert r2.meta['request'] is r2
def test_non_pickable_object(self):
q = self.queue()
try:
q.push(lambda x: x)
except ValueError as exc:
self.assertIsInstance(exc.__context__, AttributeError)
sel = Selector(text='<html><body><p>some text</p></body></html>')
try:
q.push(sel)
except ValueError as exc:
self.assertIsInstance(exc.__context__, TypeError)
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 1