From 472a8a47d0b213b45c9d3fc390628c8516650e81 Mon Sep 17 00:00:00 2001
From: Darshan Chaudhary <deathbullet@gmail.com>
Date: Sat, 21 Nov 2015 00:40:01 +0530
Subject: [PATCH] Change name, log once

---
 docs/topics/jobs.rst                |  2 ++
 docs/topics/settings.rst            | 10 ++++++++++
 scrapy/core/scheduler.py            | 10 +++++++---
 scrapy/settings/default_settings.py |  2 +-
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 303076015..093cf9bb5 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -96,4 +96,6 @@ But this will::
         somearg = response.meta['somearg']
         print "the argument passed is:", somearg
 
+If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default.
+
 .. _pickle: http://docs.python.org/library/pickle.html
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c845c59b9..d03da44c6 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1026,6 +1026,16 @@ Default: ``'scrapy.core.scheduler.Scheduler'``
 
 The scheduler to use for crawling.
 
+.. setting:: SCHEDULER_DEBUG
+
+SCHEDULER_DEBUG
+---------------
+
+Default: ``False``
+
+Setting to ``True`` will log the first unserializable request encountered.
+Stats collected.
+
 .. setting:: SPIDER_CONTRACTS
 
 SPIDER_CONTRACTS
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 8be9a0019..e0788a8c5 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -30,7 +30,7 @@ class Scheduler(object):
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
-        logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS')
+        logunser = settings.getbool('SCHEDULER_DEBUG')
         return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser,
                    stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass)
 
@@ -86,9 +86,13 @@ class Scheduler(object):
             self.dqs.push(reqd, -request.priority)
         except ValueError as e: # non serializable request
             if self.logunser:
-                logger.error("Unable to serialize request: %(request)s - reason: %(reason)s",
-                             {'request': request, 'reason': e},
+                msg = ("Unable to serialize request: %(request)s - reason: %(reason)s"
+                       " - no more unserializable requests will be logged"
+                       " (stats being collected)")
+                logger.error(msg, {'request': request, 'reason': e},
                              exc_info=True, extra={'spider': self.spider})
+                self.logunser = False
+            self.stats.inc_value('scheduler/unserializable', spider=self.spider)
             return
         else:
             return True
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 2c267b4cc..673df5129 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -191,7 +191,7 @@ LOG_STDOUT = False
 LOG_LEVEL = 'DEBUG'
 LOG_FILE = None
 
-LOG_UNSERIALIZABLE_REQUESTS = False
+SCHEDULER_DEBUG = False
 
 LOGSTATS_INTERVAL = 60.0