From 3152e66fec3e83037d92553fc3939df6d075368c Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 20 Aug 2009 10:54:58 -0300 Subject: [PATCH] deprecate domain_open signal and handle stats domain open/close directly from the engine --- docs/experimental/exporters.rst | 4 ++-- docs/topics/item-pipeline.rst | 4 ++-- docs/topics/signals.rst | 19 +++++-------------- docs/topics/stats.rst | 7 +++---- .../contrib/downloadermiddleware/httpcache.py | 2 +- .../contrib/downloadermiddleware/robotstxt.py | 4 ++-- scrapy/contrib/pipeline/media.py | 4 ++-- scrapy/contrib/webconsole/livestats.py | 4 ++-- scrapy/contrib/webconsole/spiderctl.py | 4 ++-- scrapy/contrib_exp/history/middleware.py | 2 +- scrapy/contrib_exp/pipeline/shoveitem.py | 4 ++-- scrapy/core/engine.py | 9 ++++++++- scrapy/core/signals.py | 4 +++- scrapy/stats/collector/__init__.py | 4 ---- 14 files changed, 35 insertions(+), 40 deletions(-) diff --git a/docs/experimental/exporters.rst b/docs/experimental/exporters.rst index 536f30ab6..9fd0ad8c3 100644 --- a/docs/experimental/exporters.rst +++ b/docs/experimental/exporters.rst @@ -35,10 +35,10 @@ Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline class XmlExportPipeline(object): def __init__(self): - dispatcher.connect(self.domain_open, signals.domain_open) + dispatcher.connect(self.domain_opened, signals.domain_opened) dispatcher.connect(self.domain_closed, signals.domain_closed) - def domain_open(self, domain): + def domain_opened(self, domain): self.file = open('%s_products.xml' % domain) self.exporter = XmlItemExporter(self.file) self.exporter.start_exporting() diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index e54b59737..abd48066e 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -73,10 +73,10 @@ spider returns multiples items with the same id:: class DuplicatesPipeline(object): def __init__(self): self.domaininfo = {} - dispatcher.connect(self.domain_open, signals.domain_open) + dispatcher.connect(self.domain_opened, signals.domain_opened) dispatcher.connect(self.domain_closed, signals.domain_closed) - def domain_open(self, domain): + def domain_opened(self, domain): self.duplicates[domain] = set() def domain_closed(self, domain): diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3b827c065..7866df72b 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -32,7 +32,8 @@ order. .. signal:: domain_closed .. function:: domain_closed(domain, spider, reason) - Sent right after a spider/domain has been closed. + Sent after a spider/domain has been closed. This can be used to release + per-spider resources reserved on :signal:`domain_opened`. :param domain: a string which contains the domain of the spider which has been closed @@ -50,22 +51,12 @@ order. Ctrl-C to stop it) the reason will be ``'shutdown'``. :type reason: str -.. signal:: domain_open -.. function:: domain_open(domain, spider) - - Sent right before a spider has been opened for crawling. - - :param domain: a string which contains the domain of the spider which is about - to be opened - :type domain: str - - :param spider: the spider which is about to be opened - :type spider: :class:`~scrapy.spider.BaseSpider` object - .. signal:: domain_opened .. function:: domain_opened(domain, spider) - Sent right after a spider has been opened for crawling. + Sent after a spider/domain has been opened for crawling. This is typically + used to reserve per-spider resources, but can be used for any task that + needs to be performed when a spider/domain is opened. :param domain: a string with the domain of the spider which has been opened :type domain: str diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 2afc78397..5dbbb4f10 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -67,8 +67,7 @@ Get all global stats from a given domain:: {'hostname': 'localhost', 'spiders_crawled': 8} Set domain/spider specific stat value (domains must be opened first, but this -is done automatically by the Stats Collector on the :signal:`domain_open` -signal):: +task is handled automatically by the Scrapy engine):: stats.set_value('start_time', datetime.now(), domain='example.com') @@ -170,8 +169,8 @@ class (which they all inherit from). .. method:: open_domain(domain) Open the given domain for stats collection. This method must be called - prior to working with any stats specific to that domain, but it's - called automatically when the :signal:`domain_open` signal is received. + prior to working with any stats specific to that domain, but this task + is handled automatically by the Scrapy engine. .. method:: close_domain(domain) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 18085bea7..04c6082ad 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -24,7 +24,7 @@ class HttpCacheMiddleware(object): raise NotConfigured self.cache = Cache(settings['HTTPCACHE_DIR'], sectorize=settings.getbool('HTTPCACHE_SECTORIZE')) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') - dispatcher.connect(self.open_domain, signal=signals.domain_open) + dispatcher.connect(self.open_domain, signal=signals.domain_opened) def open_domain(self, domain): self.cache.open_domain(domain) diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index f0fed6193..a0c6c98f5 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -26,7 +26,7 @@ class RobotsTxtMiddleware(object): self._spider_netlocs = {} self._useragents = {} self._pending = {} - dispatcher.connect(self.domain_open, signals.domain_open) + dispatcher.connect(self.domain_opened, signals.domain_opened) dispatcher.connect(self.domain_closed, signals.domain_closed) def process_request(self, request, spider): @@ -51,7 +51,7 @@ class RobotsTxtMiddleware(object): rp.parse(response.body.splitlines()) self._parsers[urlparse_cached(response).netloc] = rp - def domain_open(self, spider): + def domain_opened(self, spider): self._spider_netlocs[spider] = set() self._useragents[spider] = getattr(spider, 'user_agent', None) \ or settings['USER_AGENT'] diff --git a/scrapy/contrib/pipeline/media.py b/scrapy/contrib/pipeline/media.py index 326058d60..75677654c 100644 --- a/scrapy/contrib/pipeline/media.py +++ b/scrapy/contrib/pipeline/media.py @@ -22,10 +22,10 @@ class MediaPipeline(object): def __init__(self): self.domaininfo = {} - dispatcher.connect(self.domain_open, signals.domain_open) + dispatcher.connect(self.domain_opened, signals.domain_opened) dispatcher.connect(self.domain_closed, signals.domain_closed) - def domain_open(self, domain): + def domain_opened(self, domain): self.domaininfo[domain] = self.DomainInfo(domain) def domain_closed(self, domain): diff --git a/scrapy/contrib/webconsole/livestats.py b/scrapy/contrib/webconsole/livestats.py index 7b338ccea..4ba7c7549 100644 --- a/scrapy/contrib/webconsole/livestats.py +++ b/scrapy/contrib/webconsole/livestats.py @@ -20,14 +20,14 @@ class LiveStats(object): def __init__(self): self.domains = {} - dispatcher.connect(self.domain_open, signal=signals.domain_open) + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) dispatcher.connect(self.domain_closed, signal=signals.domain_closed) dispatcher.connect(self.item_scraped, signal=signals.item_scraped) dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded) dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - def domain_open(self, domain, spider): + def domain_opened(self, domain, spider): pstats = SpiderStats() self.domains[spider.domain_name] = pstats pstats.started = datetime.now().replace(microsecond=0) diff --git a/scrapy/contrib/webconsole/spiderctl.py b/scrapy/contrib/webconsole/spiderctl.py index 40224f3e9..8a167484f 100644 --- a/scrapy/contrib/webconsole/spiderctl.py +++ b/scrapy/contrib/webconsole/spiderctl.py @@ -18,13 +18,13 @@ class Spiderctl(object): def __init__(self): self.running = set() self.finished = set() - dispatcher.connect(self.domain_open, signal=signals.domain_open) + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) dispatcher.connect(self.domain_closed, signal=signals.domain_closed) from scrapy.management.web import webconsole_discover_module dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module) - def domain_open(self, domain, spider): + def domain_opened(self, domain, spider): self.running.add(domain) def domain_closed(self, domain, spider): diff --git a/scrapy/contrib_exp/history/middleware.py b/scrapy/contrib_exp/history/middleware.py index b1771e96b..32dde7913 100644 --- a/scrapy/contrib_exp/history/middleware.py +++ b/scrapy/contrib_exp/history/middleware.py @@ -22,7 +22,7 @@ class HistoryMiddleware(object): if not historycls: raise NotConfigured self.historydata = historycls() - dispatcher.connect(self.open_domain, signal=signals.domain_open) + dispatcher.connect(self.open_domain, signal=signals.domain_opened) dispatcher.connect(self.close_domain, signal=signals.domain_closed) def process_request(self, request, spider): diff --git a/scrapy/contrib_exp/pipeline/shoveitem.py b/scrapy/contrib_exp/pipeline/shoveitem.py index 5bd377769..584e211ba 100644 --- a/scrapy/contrib_exp/pipeline/shoveitem.py +++ b/scrapy/contrib_exp/pipeline/shoveitem.py @@ -24,7 +24,7 @@ class ShoveItemPipeline(object): self.opts = settings['SHOVEITEM_STORE_OPT'] or {} self.stores = {} - dispatcher.connect(self.domain_open, signal=signals.domain_open) + dispatcher.connect(self.domain_opened, signal=signals.domain_opened) dispatcher.connect(self.domain_closed, signal=signals.domain_closed) def process_item(self, domain, item): @@ -43,7 +43,7 @@ class ShoveItemPipeline(object): self.log(domain, item, status) return item - def domain_open(self, domain): + def domain_opened(self, domain): uri = Template(self.uritpl).substitute(domain=domain) self.stores[domain] = Shove(uri, **self.opts) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index bf9c84751..b356c7cc4 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -12,6 +12,7 @@ from twisted.python.failure import Failure from scrapy.xlib.pydispatch import dispatcher from scrapy import log +from scrapy.stats import stats from scrapy.conf import settings from scrapy.core import signals from scrapy.core.scheduler import Scheduler @@ -115,6 +116,7 @@ class ExecutionEngine(object): spider = spiders.fromdomain(domain) signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ domain=domain, spider=spider, reason='shutdown') + stats.close_domain(domain, reason='shutdown') for tsk, _, _ in self.tasks: # stop looping calls if tsk.running: tsk.stop() @@ -275,8 +277,11 @@ class ExecutionEngine(object): self.downloader.open_domain(domain) self.scraper.open_domain(domain) + stats.open_domain(domain) + # XXX: sent for backwards compatibility (will be removed in Scrapy 0.8) signals.send_catch_log(signals.domain_open, sender=self.__class__, domain=domain, spider=spider) + signals.send_catch_log(signals.domain_opened, sender=self.__class__, domain=domain, spider=spider) def _domain_idle(self, domain): @@ -325,7 +330,9 @@ class ExecutionEngine(object): self.scheduler.close_domain(domain) self.scraper.close_domain(domain) reason = self.closing.pop(domain, 'finished') - signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, reason=reason) + signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \ + domain=domain, spider=spider, reason=reason) + stats.close_domain(domain, reason=reason) log.msg("Domain closed (%s)" % reason, domain=domain) self._mainloop() diff --git a/scrapy/core/signals.py b/scrapy/core/signals.py index 0cdee5500..37b24e6e1 100644 --- a/scrapy/core/signals.py +++ b/scrapy/core/signals.py @@ -11,7 +11,6 @@ from scrapy import log engine_started = object() engine_stopped = object() -domain_open = object() domain_opened = object() domain_idle = object() domain_closed = object() @@ -23,6 +22,9 @@ item_scraped = object() item_passed = object() item_dropped = object() +# XXX: deprecated signals (will be removed in Scrapy 0.8) +domain_open = object() + def send_catch_log(signal, sender=None, **kwargs): """ Send a signal and log any exceptions raised by its listeners diff --git a/scrapy/stats/collector/__init__.py b/scrapy/stats/collector/__init__.py index 7fe1553af..3ffa1c06a 100644 --- a/scrapy/stats/collector/__init__.py +++ b/scrapy/stats/collector/__init__.py @@ -17,10 +17,6 @@ class StatsCollector(object): self._dump = settings.getbool('STATS_DUMP') self._stats = {None: {}} # None is for global stats - dispatcher.connect(self.open_domain, signal=signals.domain_open) - dispatcher.connect(self._start_closing_domain, signal=signals.domain_closed) - dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped) - def get_value(self, key, default=None, domain=None): return self._stats[domain].get(key, default)