From 1a545157c66a3daf69da1a1510aa1d41942f242b Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 28 Jan 2014 12:52:28 +0100 Subject: [PATCH 1/2] Offsite: add 2 stats counters --- scrapy/contrib/spidermiddleware/offsite.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index f8acbf8d8..317d6af72 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -13,9 +13,12 @@ from scrapy import log class OffsiteMiddleware(object): + def __init__(self, stats=False): + self.stats_enabled = stats + @classmethod def from_crawler(cls, crawler): - o = cls() + o = cls(stats=True) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o @@ -30,6 +33,15 @@ class OffsiteMiddleware(object): self.domains_seen.add(domain) log.msg(format="Filtered offsite request to %(domain)r: %(request)s", level=log.DEBUG, spider=spider, domain=domain, request=x) + + if self.stats_enabled: + spider.crawler.stats.inc_value('offsite/domains', + spider=spider) + + if self.stats_enabled: + spider.crawler.stats.inc_value('offsite/filtered', + spider=spider) + else: yield x From fd5b40593a4303acbc00d36a5212846a6a375952 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 29 Jan 2014 14:33:12 +0100 Subject: [PATCH 2/2] Always enable offsite stats + refactor test to initialize crawler --- scrapy/contrib/spidermiddleware/offsite.py | 17 +++++------------ scrapy/tests/test_spidermiddleware_offsite.py | 4 +++- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/scrapy/contrib/spidermiddleware/offsite.py b/scrapy/contrib/spidermiddleware/offsite.py index 317d6af72..95702d777 100644 --- a/scrapy/contrib/spidermiddleware/offsite.py +++ b/scrapy/contrib/spidermiddleware/offsite.py @@ -13,12 +13,12 @@ from scrapy import log class OffsiteMiddleware(object): - def __init__(self, stats=False): - self.stats_enabled = stats + def __init__(self, stats): + self.stats = stats @classmethod def from_crawler(cls, crawler): - o = cls(stats=True) + o = cls(crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o @@ -33,15 +33,8 @@ class OffsiteMiddleware(object): self.domains_seen.add(domain) log.msg(format="Filtered offsite request to %(domain)r: %(request)s", level=log.DEBUG, spider=spider, domain=domain, request=x) - - if self.stats_enabled: - spider.crawler.stats.inc_value('offsite/domains', - spider=spider) - - if self.stats_enabled: - spider.crawler.stats.inc_value('offsite/filtered', - spider=spider) - + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) else: yield x diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index fd6e43753..8477e4dee 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -3,13 +3,15 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware +from scrapy.utils.test import get_crawler class TestOffsiteMiddleware(TestCase): def setUp(self): self.spider = self._get_spider() - self.mw = OffsiteMiddleware() + crawler = get_crawler() + self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider) def _get_spider(self):