RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3082 from elacuesta/pickle-requests 

[MRG+1] Do not serialize unpickable objects (py3)
This commit is contained in:
Konstantin Lopuhin 2018-02-08 23:39:37 +03:00 committed by GitHub
parent eff469245a 6edd4114c4
commit 8d2240d066
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 11 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
requirements.txt
View File

@ -7,4 +7,4 @@ queuelib
six>=1.5.2
PyDispatcher>=2.0.5
service_identity
parsel>=1.1
parsel>=1.4

7
scrapy/squeues.py
View File

@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
def _pickle_serialize(obj):
try:
return pickle.dumps(obj, protocol=2)
# Python>=3.5 raises AttributeError here while
# Python<=3.4 raises pickle.PicklingError
except (pickle.PicklingError, AttributeError) as e:
# Python <= 3.4 raises pickle.PicklingError here while
# 3.5 <= Python < 3.6 raises AttributeError and
# Python >= 3.6 raises TypeError
except (pickle.PicklingError, AttributeError, TypeError) as e:
raise ValueError(str(e))
PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \

2
setup.py
View File

@ -71,7 +71,7 @@ setup(
'pyOpenSSL',
'cssselect>=0.9',
'six>=1.5.2',
'parsel>=1.1',
'parsel>=1.4',
'PyDispatcher>=2.0.5',
'service_identity',
],

7
tests/test_squeues.py
View File

@ -5,6 +5,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif
from scrapy.item import Item, Field
from scrapy.http import Request
from scrapy.loader import ItemLoader
from scrapy.selector import Selector
class TestItem(Item):
name = Field()
@ -17,20 +18,22 @@ class TestLoader(ItemLoader):
name_out = staticmethod(_test_procesor)
def nonserializable_object_test(self):
q = self.queue()
try:
pickle.dumps(lambda x: x)
except Exception:
# Trigger Twisted bug #7989
import twisted.persisted.styles # NOQA
q = self.queue()
self.assertRaises(ValueError, q.push, lambda x: x)
else:
# Use a different unpickleable object
class A(object): pass
a = A()
a.__reduce__ = a.__reduce_ex__ = None
q = self.queue()
self.assertRaises(ValueError, q.push, a)
# Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
sel = Selector(text='<html><body><p>some text</p></body></html>')
self.assertRaises(ValueError, q.push, sel)
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):