diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 093cf9bb5..55105dcfd 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -96,6 +96,8 @@ But this will:: somearg = response.meta['somearg'] print "the argument passed is:", somearg -If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default. +If you wish to log the requests that couldn't be serialized, you can set the +``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. +It is ``False`` by default. .. _pickle: http://docs.python.org/library/pickle.html diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d03da44c6..d14f230e4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1033,8 +1033,15 @@ SCHEDULER_DEBUG Default: ``False`` -Setting to ``True`` will log the first unserializable request encountered. -Stats collected. +Setting to ``True`` will log debug information about the requests scheduler. +This currently logs(only once) if the requests cannot be serialized to disk. +Stats counter (``scheduler/unserializable``) tracks the number of times this happens. + +Example entry in logs:: + + 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + - reason: - no more unserializable requests will be logged (stats being + collected) .. setting:: SPIDER_CONTRACTS diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a3cb1bcff..dcd6fb989 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -84,15 +84,16 @@ class Scheduler(object): try: reqd = request_to_dict(request, self.spider) self.dqs.push(reqd, -request.priority) - except ValueError as e: # non serializable request + except ValueError as e: # non serializable request if self.logunser: - msg = ("Unable to serialize request: %(request)s - reason: %(reason)s" - " - no more unserializable requests will be logged" - " (stats being collected)") + msg = ("Unable to serialize request: %(request)s - reason:" + " %(reason)s - no more unserializable requests will be" + " logged (stats being collected)") logger.error(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) self.logunser = False - self.stats.inc_value('scheduler/unserializable', spider=self.spider) + self.stats.inc_value('scheduler/unserializable', + spider=self.spider) return else: return True