From bd0b639b2127908816bb2b594f77f2d5210bed5b Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 10 Mar 2015 15:59:44 -0300 Subject: [PATCH] Fix logging usage across docs --- docs/topics/debug.rst | 5 +---- docs/topics/extensions.rst | 9 ++++++--- docs/topics/practices.rst | 17 ++++++++++++++--- docs/topics/request-response.rst | 4 ++-- docs/topics/spiders.rst | 10 ++++------ 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 5739d9e1b..a3e72097c 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -132,16 +132,13 @@ Logging is another useful option for getting information about your spider run. Although not as convenient, it comes with the advantage that the logs will be available in all future runs should they be necessary again:: - from scrapy import log - def parse_details(self, response): item = response.meta.get('item', None) if item: # populate more `item` fields return item else: - self.log('No item received for %s' % response.url, - level=log.WARNING) + self.logger.warning('No item received for %s', response.url) For more information, check the :ref:`topics-logging` section. diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index 8cd588c4a..19c296651 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -102,9 +102,12 @@ number of items will be specified through the ``MYEXT_ITEMCOUNT`` setting. Here is the code of such extension:: + import logging from scrapy import signals from scrapy.exceptions import NotConfigured + logger = logging.getLogger(__name__) + class SpiderOpenCloseLogging(object): def __init__(self, item_count): @@ -133,15 +136,15 @@ Here is the code of such extension:: return ext def spider_opened(self, spider): - spider.log("opened spider %s" % spider.name) + logger.info("opened spider %s", spider.name) def spider_closed(self, spider): - spider.log("closed spider %s" % spider.name) + logger.info("closed spider %s", spider.name) def item_scraped(self, item, spider): self.items_scraped += 1 if self.items_scraped % self.item_count == 0: - spider.log("scraped %d items" % self.items_scraped) + logger.info("scraped %d items", self.items_scraped) .. _topics-extensions-ref: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 2f848e6d9..6bd74c794 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -30,9 +30,12 @@ project as example. from twisted.internet import reactor from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings - runner = CrawlerRunner(get_project_settings()) + settings = get_project_settings() + configure_logging(settings) + runner = CrawlerRunner(settings) # 'followall' is the name of one of the spiders of the project. d = runner.crawl('followall', domain='scrapinghub.com') @@ -54,11 +57,13 @@ the spider class as first argument in the :meth:`CrawlerRunner.crawl from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging class MySpider(scrapy.Spider): # Your spider definition ... + configure_logging(settings) runner = CrawlerRunner({ 'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)' }) @@ -85,9 +90,12 @@ Here is an example that runs multiple spiders simultaneously, using the from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings - runner = CrawlerRunner(get_project_settings()) + settings = get_project_settings() + configure_logging(settings) + runner = CrawlerRunner(settings) dfs = set() for domain in ['scrapinghub.com', 'insophia.com']: d = runner.crawl('followall', domain=domain) @@ -102,9 +110,12 @@ Same example but running the spiders sequentially by chaining the deferreds: from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings - runner = CrawlerRunner(get_project_settings()) + settings = get_project_settings() + configure_logging(settings) + runner = CrawlerRunner(settings) @defer.inlineCallbacks def crawl(): diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 5b4ced992..1d695a5f2 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -189,7 +189,7 @@ Example:: def parse_page2(self, response): # this would log http://www.example.com/some_page.html - self.log("Visited %s" % response.url) + self.logger.info("Visited %s", response.url) In some cases you may be interested in passing arguments to those callback functions so you can receive the arguments later, in the second callback. You @@ -382,7 +382,7 @@ method for this job. Here's an example spider which uses it:: def after_login(self, response): # check login succeed before going on if "authentication failed" in response.body: - self.log("Login failed", level=log.ERROR) + self.logger.error("Login failed") return # continue scraping with authenticated session... diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index c64e98396..7c7d5d731 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -224,7 +224,7 @@ Let's see an example:: ] def parse(self, response): - self.log('A response from %s just arrived!' % response.url) + self.logger.info('A response from %s just arrived!', response.url) Return multiple Requests and items from a single callback:: @@ -412,7 +412,7 @@ Let's now take a look at an example CrawlSpider with rules:: ) def parse_item(self, response): - self.log('Hi, this is an item page! %s' % response.url) + self.logger.info('Hi, this is an item page! %s', response.url) item = scrapy.Item() item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') item['name'] = response.xpath('//td[@id="item_name"]/text()').extract() @@ -515,7 +515,6 @@ XMLFeedSpider example These spiders are pretty easy to use, let's have a look at one example:: - from scrapy import log from scrapy.contrib.spiders import XMLFeedSpider from myproject.items import TestItem @@ -527,7 +526,7 @@ These spiders are pretty easy to use, let's have a look at one example:: itertag = 'item' def parse_node(self, response, node): - log.msg('Hi, this is a <%s> node!: %s' % (self.itertag, ''.join(node.extract()))) + self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract())) item = TestItem() item['id'] = node.xpath('@id').extract() @@ -576,7 +575,6 @@ CSVFeedSpider example Let's see an example similar to the previous one, but using a :class:`CSVFeedSpider`:: - from scrapy import log from scrapy.contrib.spiders import CSVFeedSpider from myproject.items import TestItem @@ -589,7 +587,7 @@ Let's see an example similar to the previous one, but using a headers = ['id', 'name', 'description'] def parse_row(self, response, row): - log.msg('Hi, this is a row!: %r' % row) + self.logger.info('Hi, this is a row!: %r', row) item = TestItem() item['id'] = row['id']