From cca0b910007846a4eeb147c374b2baeb31ea188e Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 1 Sep 2011 19:40:44 -0300
Subject: [PATCH] add setting to enable logging when unserializable requests
 are found

---
 scrapy/core/scheduler.py            | 11 ++++++++---
 scrapy/settings/default_settings.py |  2 ++
 scrapy/squeue.py                    |  5 ++++-
 scrapy/tests/test_squeue.py         |  4 ++++
 4 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 9dc7be5c2..6d74085ea 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -13,17 +13,19 @@ from scrapy import log
 
 class Scheduler(object):
 
-    def __init__(self, dupefilter, jobdir=None, dqclass=None):
+    def __init__(self, dupefilter, jobdir=None, dqclass=None, logunser=False):
         self.df = dupefilter
         self.dqdir = join(jobdir, 'requests.queue') if jobdir else None
         self.dqclass = dqclass
+        self.logunser = logunser
 
     @classmethod
     def from_settings(cls, settings):
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
         dupefilter = dupefilter_cls.from_settings(settings)
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
-        return cls(dupefilter, job_dir(settings), dqclass)
+        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')
+        return cls(dupefilter, job_dir(settings), dqclass, logunser)
 
     def has_pending_requests(self):
         return len(self) > 0
@@ -59,7 +61,10 @@ class Scheduler(object):
         try:
             reqd = request_to_dict(request, self.spider)
             self.dqs.push(reqd, -request.priority)
-        except ValueError: # non serializable request
+        except ValueError, e: # non serializable request
+            if self.logunser:
+                log.msg("Unable to serialize request: %s - reason: %s" % \
+                    (request, str(e)), level=log.ERROR, spider=self.spider)
             return
         else:
             stats.inc_value('scheduler/disk_enqueued', spider=self.spider)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 62db9ebf2..415c7519d 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -188,6 +188,8 @@ LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
 
+LOG_UNSERIALIZABLE_REQUESTS = False
+
 LOGSTATS_INTERVAL = 60.0
 
 MAIL_DEBUG = False
diff --git a/scrapy/squeue.py b/scrapy/squeue.py
index abfe2e6db..c01919fc1 100644
--- a/scrapy/squeue.py
+++ b/scrapy/squeue.py
@@ -10,7 +10,10 @@ from scrapy.utils.queue import DiskQueue
 class PickleDiskQueue(DiskQueue):
 
     def push(self, obj):
-        s = pickle.dumps(obj, protocol=2)
+        try:
+            s = pickle.dumps(obj, protocol=2)
+        except pickle.PicklingError, e:
+            raise ValueError(str(e))
         super(PickleDiskQueue, self).push(s)
 
     def pop(self):
diff --git a/scrapy/tests/test_squeue.py b/scrapy/tests/test_squeue.py
index 958aa4afc..03fd7da87 100644
--- a/scrapy/tests/test_squeue.py
+++ b/scrapy/tests/test_squeue.py
@@ -30,6 +30,10 @@ class MarshalDiskQueueTest(t.DiskQueueTest):
         self.assertEqual(q.pop(), 123)
         self.assertEqual(q.pop(), {'a': 'dict'})
 
+    def test_nonserializable_object(self):
+        q = self.queue()
+        self.assertRaises(ValueError, q.push, lambda x: x)
+
 class ChunkSize1MarshalDiskQueueTest(MarshalDiskQueueTest):
     chunksize = 1