RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

colllect max itemproc_size and active_size in scraper per domain

This commit is contained in:
Daniel Grana 2009-08-11 15:47:26 -03:00
parent cd8eed77c5
commit 02cb1e3432
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/core/scraper.py
View File

@ -84,6 +84,8 @@ class Scraper(object):
def enqueue_scrape(self, response, request, spider):
site = self.sites[spider.domain_name]
dfd = site.add_response_request(response, request)
stats.max_value('scraper/max_active_size', \
site.active_size, 0, domain=spider.domain_name),
def finish_scraping(_):
site.finish_response(response)
self._scrape_next(spider, site)
@ -158,6 +160,8 @@ class Scraper(object):
signals.send_catch_log(signal=signals.item_scraped, sender=self.__class__, \
item=output, spider=spider, response=response)
self.sites[domain].itemproc_size += 1
stats.max_value('scraper/max_itemproc_size', \
self.sites[domain].itemproc_size, 0, domain=domain)
dfd = self.itemproc.process_item(output, spider)
dfd.addBoth(self._itemproc_finished, output, spider)
return dfd