From c5ffa1a59f87a806319f40a436010428812bbf49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Wed, 5 Aug 2015 20:33:25 -0300 Subject: [PATCH] Twisted 15.3.0 does not raises expected exception serializing lambda functions --- scrapy/_monkeypatches.py | 11 +++++++++++ tests/test_squeues.py | 4 ++++ 2 files changed, 15 insertions(+) diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index e0ae45b64..782891326 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,4 +1,5 @@ import sys +from six.moves import copyreg if sys.version_info[0] == 2: from urlparse import urlparse @@ -12,3 +13,13 @@ if sys.version_info[0] == 2: if urlparse('s3://bucket/key?key=value').query != 'key=value': from urlparse import uses_query uses_query.append('s3') + + +# Undo what Twisted's perspective broker adds to pickle register +# to prevent bugs like Twisted#7989 while serializing requests +import twisted.persisted.styles # NOQA +# Remove only entries with twisted serializers for non-twisted types. +for k, v in frozenset(copyreg.dispatch_table.items()): + if not getattr(k, '__module__', '').startswith('twisted') \ + and getattr(v, '__module__', '').startswith('twisted'): + copyreg.dispatch_table.pop(k) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index b3e9a2208..f2d7be67d 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -31,6 +31,8 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): self.assertEqual(q.pop(), {'a': 'dict'}) def test_nonserializable_object(self): + # Trigger Twisted bug #7989 + import twisted.persisted.styles # NOQA q = self.queue() self.assertRaises(ValueError, q.push, lambda x: x) @@ -109,6 +111,8 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest): self.assertEqual(q.pop(), 'a') def test_nonserializable_object(self): + # Trigger Twisted bug #7989 + import twisted.persisted.styles # NOQA q = self.queue() self.assertRaises(ValueError, q.push, lambda x: x)