From 1bc2339bb837bbc53b6f0760156df2bc60f17490 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 3 Jun 2011 01:13:00 -0300 Subject: [PATCH] Merged item passed and item scraped concepts, as they have often proved confusing in the past. This means: * original item_scraped signal was removed * original item_passed signal was renamed to item_scraped * old log lines "Scraped Item..." removed * old log lines "Passed Item..." renamed to "Scraped Item..." --- docs/topics/signals.rst | 33 +++++-------------------------- scrapy/contrib/closespider.py | 4 ++-- scrapy/contrib/corestats.py | 5 ----- scrapy/contrib/feedexport.py | 4 ++-- scrapy/core/scraper.py | 20 ++++++++----------- scrapy/logformatter.py | 9 +++------ scrapy/signals.py | 3 ++- scrapy/tests/test_logformatter.py | 3 ++- 8 files changed, 24 insertions(+), 57 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 6343f02f3..f6c600cad 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -63,45 +63,22 @@ item_scraped ------------ .. signal:: item_scraped -.. function:: item_scraped(item, spider, response) +.. function:: item_scraped(item, response, spider) - Sent when the engine receives a new scraped item from the spider, and right - before the item is sent to the :ref:`topics-item-pipeline`. + Sent when an item has been scraped, after it has passed all the + :ref:`topics-item-pipeline` stages (without being dropped). This signal supports returning deferreds from their handlers. - :param item: is the item scraped + :param item: the item scraped :type item: :class:`~scrapy.item.Item` object - :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.spider.BaseSpider` object - - :param response: the response from which the item was scraped + :param response: the response from where the item was scraped :type response: :class:`~scrapy.http.Response` object -item_passed ------------ - -.. signal:: item_passed -.. function:: item_passed(item, spider, original_item) - - Sent after an item has passed all the :ref:`topics-item-pipeline` stages - without being dropped. Same as :func:`item_scraped` if there are no - pipelines enabled. - - This signal supports returning deferreds from their handlers. - - :param item: the item which passed the pipeline - :type item: :class:`~scrapy.item.Item` object - :param spider: the spider which scraped the item :type spider: :class:`~scrapy.spider.BaseSpider` object - :param original_item: the input of the item pipeline. This is typically the - same :class:`~scrapy.item.Item` object received in the ``item`` - parameter, unless some pipeline stage created a new item. - :type spider: :class:`~scrapy.spider.BaseSpider` object - item_dropped ------------ diff --git a/scrapy/contrib/closespider.py b/scrapy/contrib/closespider.py index 49fe6ecb7..a30605eee 100644 --- a/scrapy/contrib/closespider.py +++ b/scrapy/contrib/closespider.py @@ -34,7 +34,7 @@ class CloseSpider(object): if self.timeout: dispatcher.connect(self.spider_opened, signal=signals.spider_opened) if self.itempassed: - dispatcher.connect(self.item_passed, signal=signals.item_passed) + dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.spider_closed, signal=signals.spider_closed) def catch_log(self, event): @@ -55,7 +55,7 @@ class CloseSpider(object): crawler.engine.close_spider, spider=spider, \ reason='closespider_timeout') - def item_passed(self, item, spider): + def item_scraped(self, item, spider): self.counts[spider] += 1 if self.counts[spider] == self.itempassed: crawler.engine.close_spider(spider, 'closespider_itempassed') diff --git a/scrapy/contrib/corestats.py b/scrapy/contrib/corestats.py index 807a05900..7ab080735 100644 --- a/scrapy/contrib/corestats.py +++ b/scrapy/contrib/corestats.py @@ -24,7 +24,6 @@ class CoreStats(object): dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened) dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing) dispatcher.connect(self.item_scraped, signal=signals.item_scraped) - dispatcher.connect(self.item_passed, signal=signals.item_passed) dispatcher.connect(self.item_dropped, signal=signals.item_dropped) def stats_spider_opened(self, spider): @@ -40,10 +39,6 @@ class CoreStats(object): stats.inc_value('item_scraped_count', spider=spider) stats.inc_value('item_scraped_count') - def item_passed(self, item, spider): - stats.inc_value('item_passed_count', spider=spider) - stats.inc_value('item_passed_count') - def item_dropped(self, item, spider, exception): reason = exception.__class__.__name__ stats.inc_value('item_dropped_count', spider=spider) diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index 4a04b36de..a1e215367 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -136,7 +136,7 @@ class FeedExporter(object): self.slots = {} dispatcher.connect(self.open_spider, signals.spider_opened) dispatcher.connect(self.close_spider, signals.spider_closed) - dispatcher.connect(self.item_passed, signals.item_passed) + dispatcher.connect(self.item_scraped, signals.item_scraped) def open_spider(self, spider): file = TemporaryFile(prefix='feed-') @@ -163,7 +163,7 @@ class FeedExporter(object): d.addBoth(lambda _: slot.file.close()) return d - def item_passed(self, item, spider): + def item_scraped(self, item, spider): slot = self.slots[spider] slot.exporter.export_item(item) slot.itemcount += 1 diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 1550c344f..0235a21bc 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -169,13 +169,9 @@ class Scraper(object): spider=spider) self.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): - log.msg(log.formatter.scraped(output, request, response, spider), \ - level=log.DEBUG, spider=spider) self.sites[spider].itemproc_size += 1 - dfd = send_catch_log_deferred(signal=signals.item_scraped, \ - item=output, spider=spider, response=response) - dfd.addBoth(lambda _: self.itemproc.process_item(output, spider)) - dfd.addBoth(self._itemproc_finished, output, spider) + dfd = self.itemproc.process_item(output, spider) + dfd.addBoth(self._itemproc_finished, output, response, spider) return dfd elif output is None: pass @@ -198,22 +194,22 @@ class Scraper(object): else: return spider_failure # exceptions raised in the spider code - def _itemproc_finished(self, output, item, spider): + def _itemproc_finished(self, output, item, response, spider): """ItemProcessor finished for the given ``item`` and returned ``output`` """ self.sites[spider].itemproc_size -= 1 if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): - log.msg(log.formatter.dropped(item, ex, spider), \ + log.msg(log.formatter.dropped(item, ex, response, spider), \ level=log.WARNING, spider=spider) return send_catch_log_deferred(signal=signals.item_dropped, \ item=item, spider=spider, exception=output.value) else: log.err(output, 'Error processing %s' % item, spider=spider) else: - log.msg(log.formatter.passed(output, spider), log.INFO, spider=spider) - # TODO: remove item_passed 'output' parameter for Scrapy 0.12 - return send_catch_log_deferred(signal=signals.item_passed, \ - item=output, spider=spider, output=output, original_item=item) + log.msg(log.formatter.scraped(output, response, spider), \ + log.DEBUG, spider=spider) + return send_catch_log_deferred(signal=signals.item_scraped, \ + item=output, response=response, spider=spider) diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 7e7ba01b2..4fd16c47c 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -11,11 +11,8 @@ class LogFormatter(object): return "Crawled (%d) %s (referer: %s)%s" % (response.status, \ request, referer, flags) - def scraped(self, item, request, response, spider): - return "Scraped %s in <%s>" % (item, request.url) + def scraped(self, item, response, spider): + return "Scraped %s in <%s>" % (item, response.url) - def dropped(self, item, exception, spider): + def dropped(self, item, exception, response, spider): return "Dropped %s - %s" % (item, unicode(exception)) - - def passed(self, item, spider): - return "Passed %s" % item diff --git a/scrapy/signals.py b/scrapy/signals.py index c08a11382..d2708d588 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -15,8 +15,9 @@ request_received = object() response_received = object() response_downloaded = object() item_scraped = object() -item_passed = object() item_dropped = object() stats_spider_opened = object() stats_spider_closing = object() stats_spider_closed = object() + +item_passed = item_scraped # for backwards compatibility diff --git a/scrapy/tests/test_logformatter.py b/scrapy/tests/test_logformatter.py index eb386bb57..cfa608f3e 100644 --- a/scrapy/tests/test_logformatter.py +++ b/scrapy/tests/test_logformatter.py @@ -25,7 +25,8 @@ class LoggingContribTest(unittest.TestCase): def test_dropped(self): item = {} exception = Exception(u"\u2018") - self.assertEqual(self.formatter.dropped(item, exception, self.spider), + response = Response("http://www.example.com") + self.assertEqual(self.formatter.dropped(item, exception, response, self.spider), u"Dropped {} - \u2018") if __name__ == "__main__":