RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merged item passed and item scraped concepts, as they have often proved 

confusing in the past.

This means:

* original item_scraped signal was removed
* original item_passed signal was renamed to item_scraped
* old log lines "Scraped Item..." removed
* old log lines "Passed Item..." renamed to "Scraped Item..."
This commit is contained in:
Pablo Hoffman 2011-06-03 01:13:00 -03:00
parent e6091df551
commit 1bc2339bb8
8 changed files with 24 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
docs/topics/signals.rst
View File

@ -63,45 +63,22 @@ item_scraped
------------
.. signal:: item_scraped
.. function:: item_scraped(item, spider, response)
.. function:: item_scraped(item, response, spider)
Sent when the engine receives a new scraped item from the spider, and right
before the item is sent to the :ref:`topics-item-pipeline`.
Sent when an item has been scraped, after it has passed all the
:ref:`topics-item-pipeline` stages (without being dropped).
This signal supports returning deferreds from their handlers.
:param item: is the item scraped
:param item: the item scraped
:type item: :class:`~scrapy.item.Item` object
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spider.BaseSpider` object
:param response: the response from which the item was scraped
:param response: the response from where the item was scraped
:type response: :class:`~scrapy.http.Response` object
item_passed
-----------
.. signal:: item_passed
.. function:: item_passed(item, spider, original_item)
Sent after an item has passed all the :ref:`topics-item-pipeline` stages
without being dropped. Same as :func:`item_scraped` if there are no
pipelines enabled.
This signal supports returning deferreds from their handlers.
:param item: the item which passed the pipeline
:type item: :class:`~scrapy.item.Item` object
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.spider.BaseSpider` object
:param original_item: the input of the item pipeline. This is typically the
same :class:`~scrapy.item.Item` object received in the ``item``
parameter, unless some pipeline stage created a new item.
:type spider: :class:`~scrapy.spider.BaseSpider` object
item_dropped
------------

4
scrapy/contrib/closespider.py
View File

@ -34,7 +34,7 @@ class CloseSpider(object):
if self.timeout:
dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
if self.itempassed:
dispatcher.connect(self.item_passed, signal=signals.item_passed)
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
def catch_log(self, event):
@ -55,7 +55,7 @@ class CloseSpider(object):
crawler.engine.close_spider, spider=spider, \
reason='closespider_timeout')
def item_passed(self, item, spider):
def item_scraped(self, item, spider):
self.counts[spider] += 1
if self.counts[spider] == self.itempassed:
crawler.engine.close_spider(spider, 'closespider_itempassed')

5
scrapy/contrib/corestats.py
View File

@ -24,7 +24,6 @@ class CoreStats(object):
dispatcher.connect(self.stats_spider_opened, signal=signals.stats_spider_opened)
dispatcher.connect(self.stats_spider_closing, signal=signals.stats_spider_closing)
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.item_passed, signal=signals.item_passed)
dispatcher.connect(self.item_dropped, signal=signals.item_dropped)
def stats_spider_opened(self, spider):
@ -40,10 +39,6 @@ class CoreStats(object):
stats.inc_value('item_scraped_count', spider=spider)
stats.inc_value('item_scraped_count')
def item_passed(self, item, spider):
stats.inc_value('item_passed_count', spider=spider)
stats.inc_value('item_passed_count')
def item_dropped(self, item, spider, exception):
reason = exception.__class__.__name__
stats.inc_value('item_dropped_count', spider=spider)

4
scrapy/contrib/feedexport.py
View File

@ -136,7 +136,7 @@ class FeedExporter(object):
self.slots = {}
dispatcher.connect(self.open_spider, signals.spider_opened)
dispatcher.connect(self.close_spider, signals.spider_closed)
dispatcher.connect(self.item_passed, signals.item_passed)
dispatcher.connect(self.item_scraped, signals.item_scraped)
def open_spider(self, spider):
file = TemporaryFile(prefix='feed-')
@ -163,7 +163,7 @@ class FeedExporter(object):
d.addBoth(lambda _: slot.file.close())
return d
def item_passed(self, item, spider):
def item_scraped(self, item, spider):
slot = self.slots[spider]
slot.exporter.export_item(item)
slot.itemcount += 1

20
scrapy/core/scraper.py
View File

@ -169,13 +169,9 @@ class Scraper(object):
spider=spider)
self.engine.crawl(request=output, spider=spider)
elif isinstance(output, BaseItem):
log.msg(log.formatter.scraped(output, request, response, spider), \
level=log.DEBUG, spider=spider)
self.sites[spider].itemproc_size += 1
dfd = send_catch_log_deferred(signal=signals.item_scraped, \
item=output, spider=spider, response=response)
dfd.addBoth(lambda _: self.itemproc.process_item(output, spider))
dfd.addBoth(self._itemproc_finished, output, spider)
dfd = self.itemproc.process_item(output, spider)
dfd.addBoth(self._itemproc_finished, output, response, spider)
return dfd
elif output is None:
pass
@ -198,22 +194,22 @@ class Scraper(object):
else:
return spider_failure # exceptions raised in the spider code
def _itemproc_finished(self, output, item, spider):
def _itemproc_finished(self, output, item, response, spider):
"""ItemProcessor finished for the given ``item`` and returned ``output``
"""
self.sites[spider].itemproc_size -= 1
if isinstance(output, Failure):
ex = output.value
if isinstance(ex, DropItem):
log.msg(log.formatter.dropped(item, ex, spider), \
log.msg(log.formatter.dropped(item, ex, response, spider), \
level=log.WARNING, spider=spider)
return send_catch_log_deferred(signal=signals.item_dropped, \
item=item, spider=spider, exception=output.value)
else:
log.err(output, 'Error processing %s' % item, spider=spider)
else:
log.msg(log.formatter.passed(output, spider), log.INFO, spider=spider)
# TODO: remove item_passed 'output' parameter for Scrapy 0.12
return send_catch_log_deferred(signal=signals.item_passed, \
item=output, spider=spider, output=output, original_item=item)
log.msg(log.formatter.scraped(output, response, spider), \
log.DEBUG, spider=spider)
return send_catch_log_deferred(signal=signals.item_scraped, \
item=output, response=response, spider=spider)

9
scrapy/logformatter.py
View File

@ -11,11 +11,8 @@ class LogFormatter(object):
return "Crawled (%d) %s (referer: %s)%s" % (response.status, \
request, referer, flags)
def scraped(self, item, request, response, spider):
return "Scraped %s in <%s>" % (item, request.url)
def scraped(self, item, response, spider):
return "Scraped %s in <%s>" % (item, response.url)
def dropped(self, item, exception, spider):
def dropped(self, item, exception, response, spider):
return "Dropped %s - %s" % (item, unicode(exception))
def passed(self, item, spider):
return "Passed %s" % item

3
scrapy/signals.py
View File

@ -15,8 +15,9 @@ request_received = object()
response_received = object()
response_downloaded = object()
item_scraped = object()
item_passed = object()
item_dropped = object()
stats_spider_opened = object()
stats_spider_closing = object()
stats_spider_closed = object()
item_passed = item_scraped # for backwards compatibility

3
scrapy/tests/test_logformatter.py
View File

@ -25,7 +25,8 @@ class LoggingContribTest(unittest.TestCase):
def test_dropped(self):
item = {}
exception = Exception(u"\u2018")
self.assertEqual(self.formatter.dropped(item, exception, self.spider),
response = Response("http://www.example.com")
self.assertEqual(self.formatter.dropped(item, exception, response, self.spider),
u"Dropped {} - \u2018")
if __name__ == "__main__":