From 6d1ffa7137293d6f72a14ae01007807268c12df5 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 25 May 2009 20:14:50 -0300 Subject: [PATCH] renamed CrawlDebug downloader middleware to DebugMiddleware --- docs/ref/downloader-middleware.rst | 20 +++++++++++-- docs/ref/settings.rst | 1 - scrapy/conf/default_settings.py | 1 - scrapy/contrib/downloadermiddleware/debug.py | 30 ++++++++++++-------- 4 files changed, 35 insertions(+), 17 deletions(-) diff --git a/docs/ref/downloader-middleware.rst b/docs/ref/downloader-middleware.rst index cb91c6932..42fb23d92 100644 --- a/docs/ref/downloader-middleware.rst +++ b/docs/ref/downloader-middleware.rst @@ -18,10 +18,24 @@ DefaultHeadersMiddleware .. module:: scrapy.contrib.downloadermiddleware.defaultheaders :synopsis: Default Headers Downloader Middleware -.. class:: scrapy.contrib.downloadermiddleware.defaultheaders.DefaultHeadersMiddleware +.. class:: DefaultHeadersMiddleware -This middleware sets all default requests headers specified in the -:setting:`DEFAULT_REQUEST_HEADERS` setting. + This middleware sets all default requests headers specified in the + :setting:`DEFAULT_REQUEST_HEADERS` setting. + +DebugMiddleware +--------------- + +.. module:: scrapy.contrib.downloadermiddleware.debug + :synopsis: Downloader middlewares for debugging + +.. class:: DebugMiddleware + + This is a convenient middleware to inspect what's passing through the + downloader middleware. It logs all requests and responses catched by the + middleware component methods. This middleware does not use any settings and + does not come enabled by default. Instead, it's meant to be inserted at the + point of the middleware that you want to inspect. HttpCacheMiddleware ------------------- diff --git a/docs/ref/settings.rst b/docs/ref/settings.rst index e1714eadd..c46f2be6e 100644 --- a/docs/ref/settings.rst +++ b/docs/ref/settings.rst @@ -333,7 +333,6 @@ Default:: 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, - 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, } diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 191c3e1e8..742a15237 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -72,7 +72,6 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.redirect.RedirectMiddleware': 600, 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, - 'scrapy.contrib.downloadermiddleware.debug.CrawlDebug': 840, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, 'scrapy.contrib.downloadermiddleware.cache.HttpCacheMiddleware': 900, # Downloader side diff --git a/scrapy/contrib/downloadermiddleware/debug.py b/scrapy/contrib/downloadermiddleware/debug.py index 7e929fe18..d6f6922e9 100644 --- a/scrapy/contrib/downloadermiddleware/debug.py +++ b/scrapy/contrib/downloadermiddleware/debug.py @@ -1,18 +1,24 @@ from scrapy import log from scrapy.core.exceptions import NotConfigured +class DebugMiddleware(object): + + def process_request(self, request, spider): + log.msg("process_request %r" % request, domain=spider.domain_name, level=log.DEBUG) + + def process_exception(self, request, exception, spider): + log.msg("process_exception %s in %r" % (exception, request), domain=spider.domain_name, level=log.DEBUG) + + def process_response(self, request, response, spider): + log.msg("process_response %s from %r" % (response, request), domain=spider.domain_name, level=log.DEBUG) + return response + +# FIXME: backwards compatibility - will be removed before 0.7 release + +import warnings + class CrawlDebug(object): def __init__(self): - raise NotConfigured - - def process_request(self, request, spider): - log.msg("Crawling %s" % repr(request), domain=spider.domain_name, level=log.DEBUG) - - def process_exception(self, request, exception, spider): - log.msg("Crawl exception %s in %s" % (exception, repr(request)), domain=spider.domain_name, level=log.DEBUG) - - def process_response(self, request, response, spider): - log.msg("Fetched %s from %s" % (response, repr(request)), domain=spider.domain_name, level=log.DEBUG) - return response - + warnings.warn("scrapy.contrib.downloadermiddleware.debug.CrawlDebug has been replaced by scrapy.contrib.downloadermiddleware.debug.DebugDownloaderMiddleware") + DebugMiddleware.__init__(self)