From 472a8a47d0b213b45c9d3fc390628c8516650e81 Mon Sep 17 00:00:00 2001 From: Darshan Chaudhary Date: Sat, 21 Nov 2015 00:40:01 +0530 Subject: [PATCH 1/4] Change name, log once --- docs/topics/jobs.rst | 2 ++ docs/topics/settings.rst | 10 ++++++++++ scrapy/core/scheduler.py | 10 +++++++--- scrapy/settings/default_settings.py | 2 +- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 303076015..093cf9bb5 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -96,4 +96,6 @@ But this will:: somearg = response.meta['somearg'] print "the argument passed is:", somearg +If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default. + .. _pickle: http://docs.python.org/library/pickle.html diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index c845c59b9..d03da44c6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1026,6 +1026,16 @@ Default: ``'scrapy.core.scheduler.Scheduler'`` The scheduler to use for crawling. +.. setting:: SCHEDULER_DEBUG + +SCHEDULER_DEBUG +--------------- + +Default: ``False`` + +Setting to ``True`` will log the first unserializable request encountered. +Stats collected. + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 8be9a0019..e0788a8c5 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -30,7 +30,7 @@ class Scheduler(object): pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS') + logunser = settings.getbool('SCHEDULER_DEBUG') return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass) @@ -86,9 +86,13 @@ class Scheduler(object): self.dqs.push(reqd, -request.priority) except ValueError as e: # non serializable request if self.logunser: - logger.error("Unable to serialize request: %(request)s - reason: %(reason)s", - {'request': request, 'reason': e}, + msg = ("Unable to serialize request: %(request)s - reason: %(reason)s" + " - no more unserializable requests will be logged" + " (stats being collected)") + logger.error(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) + self.logunser = False + self.stats.inc_value('scheduler/unserializable', spider=self.spider) return else: return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 2c267b4cc..673df5129 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -191,7 +191,7 @@ LOG_STDOUT = False LOG_LEVEL = 'DEBUG' LOG_FILE = None -LOG_UNSERIALIZABLE_REQUESTS = False +SCHEDULER_DEBUG = False LOGSTATS_INTERVAL = 60.0 From 2d9e5937c6cd40e7fdafeddd5e54242815763b9a Mon Sep 17 00:00:00 2001 From: Darshan Chaudhary Date: Wed, 25 Nov 2015 12:34:11 +0530 Subject: [PATCH 2/4] Include deprecated warning --- scrapy/core/scheduler.py | 2 +- scrapy/settings/deprecated.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index e0788a8c5..a3cb1bcff 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -30,7 +30,7 @@ class Scheduler(object): pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('SCHEDULER_DEBUG') + logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG')) return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass) diff --git a/scrapy/settings/deprecated.py b/scrapy/settings/deprecated.py index c20c35c9c..91ed689e8 100644 --- a/scrapy/settings/deprecated.py +++ b/scrapy/settings/deprecated.py @@ -14,6 +14,7 @@ DEPRECATED_SETTINGS = [ ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), ('AUTOTHROTTLE_MAX_CONCURRENCY', 'use CONCURRENT_REQUESTS_PER_DOMAIN instead'), ('REDIRECT_MAX_METAREFRESH_DELAY', 'use METAREFRESH_MAXDELAY instead'), + ('LOG_UNSERIALIZABLE_REQUESTS', 'use SCHEDULER_DEBUG instead'), ] From 0c77b6d033a01a6279734e1626ca7e49dc8d0784 Mon Sep 17 00:00:00 2001 From: darshanime Date: Mon, 25 Jul 2016 17:55:05 +0530 Subject: [PATCH 3/4] update docs for settings --- docs/topics/jobs.rst | 4 +++- docs/topics/settings.rst | 11 +++++++++-- scrapy/core/scheduler.py | 11 ++++++----- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 093cf9bb5..55105dcfd 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -96,6 +96,8 @@ But this will:: somearg = response.meta['somearg'] print "the argument passed is:", somearg -If you wish to log the requests that couldn't be serialized, you can set the ``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. It is ``False`` by default. +If you wish to log the requests that couldn't be serialized, you can set the +``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. +It is ``False`` by default. .. _pickle: http://docs.python.org/library/pickle.html diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d03da44c6..d14f230e4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1033,8 +1033,15 @@ SCHEDULER_DEBUG Default: ``False`` -Setting to ``True`` will log the first unserializable request encountered. -Stats collected. +Setting to ``True`` will log debug information about the requests scheduler. +This currently logs(only once) if the requests cannot be serialized to disk. +Stats counter (``scheduler/unserializable``) tracks the number of times this happens. + +Example entry in logs:: + + 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + - reason: - no more unserializable requests will be logged (stats being + collected) .. setting:: SPIDER_CONTRACTS diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a3cb1bcff..dcd6fb989 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -84,15 +84,16 @@ class Scheduler(object): try: reqd = request_to_dict(request, self.spider) self.dqs.push(reqd, -request.priority) - except ValueError as e: # non serializable request + except ValueError as e: # non serializable request if self.logunser: - msg = ("Unable to serialize request: %(request)s - reason: %(reason)s" - " - no more unserializable requests will be logged" - " (stats being collected)") + msg = ("Unable to serialize request: %(request)s - reason:" + " %(reason)s - no more unserializable requests will be" + " logged (stats being collected)") logger.error(msg, {'request': request, 'reason': e}, exc_info=True, extra={'spider': self.spider}) self.logunser = False - self.stats.inc_value('scheduler/unserializable', spider=self.spider) + self.stats.inc_value('scheduler/unserializable', + spider=self.spider) return else: return True From d8e62e660db231446a430a2e59c660bced8e4a3e Mon Sep 17 00:00:00 2001 From: darshanime Date: Tue, 26 Jul 2016 20:46:12 +0530 Subject: [PATCH 4/4] update log demo print --- docs/topics/jobs.rst | 2 +- docs/topics/settings.rst | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 55105dcfd..4f9e38086 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -97,7 +97,7 @@ But this will:: print "the argument passed is:", somearg If you wish to log the requests that couldn't be serialized, you can set the -``SCHEDULER_DEBUG`` setting to ``True`` in the project's settings page. +:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. It is ``False`` by default. .. _pickle: http://docs.python.org/library/pickle.html diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index d14f230e4..e55c8bb4b 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1034,14 +1034,15 @@ SCHEDULER_DEBUG Default: ``False`` Setting to ``True`` will log debug information about the requests scheduler. -This currently logs(only once) if the requests cannot be serialized to disk. +This currently logs (only once) if the requests cannot be serialized to disk. Stats counter (``scheduler/unserializable``) tracks the number of times this happens. Example entry in logs:: - 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: - - reason: - no more unserializable requests will be logged (stats being - collected) + 1956-01-31 00:00:00+0800 [scrapy] ERROR: Unable to serialize request: + - reason: cannot serialize + (type Request)> - no more unserializable requests will be logged + (see 'scheduler/unserializable' stats counter) .. setting:: SPIDER_CONTRACTS