diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 303076015..093cf9bb5 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -96,4 +96,6 @@ But this will:: somearg = response.meta['somearg'] print "the argument passed is:", somearg +If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default. + .. _pickle: http://docs.python.org/library/pickle.html diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c845c59b9..d03da44c6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1026,6 +1026,16 @@ Default: ``'scrapy.core.scheduler.Scheduler'`` The scheduler to use for crawling. +.. setting:: SCHEDULER_DEBUG + +SCHEDULER_DEBUG +--------------- + +Default: ``False`` + +Setting to ``True`` will log the first unserializable request encountered. +Stats collected. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 8be9a0019..e0788a8c5 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -30,7 +30,7 @@ class Scheduler(object): pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS') + logunser = settings.getbool('SCHEDULER_DEBUG') return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass) @@ -86,9 +86,13 @@ class Scheduler(object): self.dqs.push(reqd, -request.priority) except ValueError as e: # non serializable request if self.logunser: - logger.error("Unable to serialize request: %(request)s - reason: %(reason)s", - {'request': request, 'reason': e}, + msg = ("Unable to serialize request: %(request)s - reason: %(reason)s" + " - no more unserializable requests will be logged" + " (stats being collected)") + logger.error(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) + self.logunser = False + self.stats.inc_value('scheduler/unserializable', spider=self.spider) return else: return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2c267b4cc..673df5129 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -191,7 +191,7 @@ LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None -LOG_UNSERIALIZABLE_REQUESTS = False +SCHEDULER_DEBUG = False LOGSTATS_INTERVAL = 60.0