RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

deprecate domain_open signal and handle stats domain open/close directly from the engine

This commit is contained in:
Pablo Hoffman 2009-08-20 10:54:58 -03:00
parent 5522560d36
commit 3152e66fec
14 changed files with 35 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/experimental/exporters.rst
View File

@ -35,10 +35,10 @@ Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline
class XmlExportPipeline(object):
def __init__(self):
dispatcher.connect(self.domain_open, signals.domain_open)
dispatcher.connect(self.domain_opened, signals.domain_opened)
dispatcher.connect(self.domain_closed, signals.domain_closed)
def domain_open(self, domain):
def domain_opened(self, domain):
self.file = open('%s_products.xml' % domain)
self.exporter = XmlItemExporter(self.file)
self.exporter.start_exporting()

4
docs/topics/item-pipeline.rst
View File

@ -73,10 +73,10 @@ spider returns multiples items with the same id::
class DuplicatesPipeline(object):
def __init__(self):
self.domaininfo = {}
dispatcher.connect(self.domain_open, signals.domain_open)
dispatcher.connect(self.domain_opened, signals.domain_opened)
dispatcher.connect(self.domain_closed, signals.domain_closed)
def domain_open(self, domain):
def domain_opened(self, domain):
self.duplicates[domain] = set()
def domain_closed(self, domain):

19
docs/topics/signals.rst
View File

@ -32,7 +32,8 @@ order.
.. signal:: domain_closed
.. function:: domain_closed(domain, spider, reason)
Sent right after a spider/domain has been closed.
Sent after a spider/domain has been closed. This can be used to release
per-spider resources reserved on :signal:`domain_opened`.
:param domain: a string which contains the domain of the spider which has
been closed
@ -50,22 +51,12 @@ order.
Ctrl-C to stop it) the reason will be ``'shutdown'``.
:type reason: str
.. signal:: domain_open
.. function:: domain_open(domain, spider)
Sent right before a spider has been opened for crawling.
:param domain: a string which contains the domain of the spider which is about
to be opened
:type domain: str
:param spider: the spider which is about to be opened
:type spider: :class:`~scrapy.spider.BaseSpider` object
.. signal:: domain_opened
.. function:: domain_opened(domain, spider)
Sent right after a spider has been opened for crawling.
Sent after a spider/domain has been opened for crawling. This is typically
used to reserve per-spider resources, but can be used for any task that
needs to be performed when a spider/domain is opened.
:param domain: a string with the domain of the spider which has been opened
:type domain: str

7
docs/topics/stats.rst
View File

@ -67,8 +67,7 @@ Get all global stats from a given domain::
{'hostname': 'localhost', 'spiders_crawled': 8}
Set domain/spider specific stat value (domains must be opened first, but this
is done automatically by the Stats Collector on the :signal:`domain_open`
signal)::
task is handled automatically by the Scrapy engine)::
stats.set_value('start_time', datetime.now(), domain='example.com')
@ -170,8 +169,8 @@ class (which they all inherit from).
.. method:: open_domain(domain)
Open the given domain for stats collection. This method must be called
prior to working with any stats specific to that domain, but it's
called automatically when the :signal:`domain_open` signal is received.
prior to working with any stats specific to that domain, but this task
is handled automatically by the Scrapy engine.
.. method:: close_domain(domain)

2
scrapy/contrib/downloadermiddleware/httpcache.py
View File

@ -24,7 +24,7 @@ class HttpCacheMiddleware(object):
raise NotConfigured
self.cache = Cache(settings['HTTPCACHE_DIR'], sectorize=settings.getbool('HTTPCACHE_SECTORIZE'))
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
dispatcher.connect(self.open_domain, signal=signals.domain_open)
dispatcher.connect(self.open_domain, signal=signals.domain_opened)
def open_domain(self, domain):
self.cache.open_domain(domain)

4
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -26,7 +26,7 @@ class RobotsTxtMiddleware(object):
self._spider_netlocs = {}
self._useragents = {}
self._pending = {}
dispatcher.connect(self.domain_open, signals.domain_open)
dispatcher.connect(self.domain_opened, signals.domain_opened)
dispatcher.connect(self.domain_closed, signals.domain_closed)
def process_request(self, request, spider):
@ -51,7 +51,7 @@ class RobotsTxtMiddleware(object):
rp.parse(response.body.splitlines())
self._parsers[urlparse_cached(response).netloc] = rp
def domain_open(self, spider):
def domain_opened(self, spider):
self._spider_netlocs[spider] = set()
self._useragents[spider] = getattr(spider, 'user_agent', None) \
or settings['USER_AGENT']

4
scrapy/contrib/pipeline/media.py
View File

@ -22,10 +22,10 @@ class MediaPipeline(object):
def __init__(self):
self.domaininfo = {}
dispatcher.connect(self.domain_open, signals.domain_open)
dispatcher.connect(self.domain_opened, signals.domain_opened)
dispatcher.connect(self.domain_closed, signals.domain_closed)
def domain_open(self, domain):
def domain_opened(self, domain):
self.domaininfo[domain] = self.DomainInfo(domain)
def domain_closed(self, domain):

4
scrapy/contrib/webconsole/livestats.py
View File

@ -20,14 +20,14 @@ class LiveStats(object):
def __init__(self):
self.domains = {}
dispatcher.connect(self.domain_open, signal=signals.domain_open)
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
dispatcher.connect(self.response_downloaded, signal=signals.response_downloaded)
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
def domain_open(self, domain, spider):
def domain_opened(self, domain, spider):
pstats = SpiderStats()
self.domains[spider.domain_name] = pstats
pstats.started = datetime.now().replace(microsecond=0)

4
scrapy/contrib/webconsole/spiderctl.py
View File

@ -18,13 +18,13 @@ class Spiderctl(object):
def __init__(self):
self.running = set()
self.finished = set()
dispatcher.connect(self.domain_open, signal=signals.domain_open)
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
from scrapy.management.web import webconsole_discover_module
dispatcher.connect(self.webconsole_discover_module, signal=webconsole_discover_module)
def domain_open(self, domain, spider):
def domain_opened(self, domain, spider):
self.running.add(domain)
def domain_closed(self, domain, spider):

2
scrapy/contrib_exp/history/middleware.py
View File

@ -22,7 +22,7 @@ class HistoryMiddleware(object):
if not historycls:
raise NotConfigured
self.historydata = historycls()
dispatcher.connect(self.open_domain, signal=signals.domain_open)
dispatcher.connect(self.open_domain, signal=signals.domain_opened)
dispatcher.connect(self.close_domain, signal=signals.domain_closed)
def process_request(self, request, spider):

4
scrapy/contrib_exp/pipeline/shoveitem.py
View File

@ -24,7 +24,7 @@ class ShoveItemPipeline(object):
self.opts = settings['SHOVEITEM_STORE_OPT'] or {}
self.stores = {}
dispatcher.connect(self.domain_open, signal=signals.domain_open)
dispatcher.connect(self.domain_opened, signal=signals.domain_opened)
dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
def process_item(self, domain, item):
@ -43,7 +43,7 @@ class ShoveItemPipeline(object):
self.log(domain, item, status)
return item
def domain_open(self, domain):
def domain_opened(self, domain):
uri = Template(self.uritpl).substitute(domain=domain)
self.stores[domain] = Shove(uri, **self.opts)

9
scrapy/core/engine.py
View File

@ -12,6 +12,7 @@ from twisted.python.failure import Failure
from scrapy.xlib.pydispatch import dispatcher
from scrapy import log
from scrapy.stats import stats
from scrapy.conf import settings
from scrapy.core import signals
from scrapy.core.scheduler import Scheduler
@ -115,6 +116,7 @@ class ExecutionEngine(object):
spider = spiders.fromdomain(domain)
signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
domain=domain, spider=spider, reason='shutdown')
stats.close_domain(domain, reason='shutdown')
for tsk, _, _ in self.tasks: # stop looping calls
if tsk.running:
tsk.stop()
@ -275,8 +277,11 @@ class ExecutionEngine(object):
self.downloader.open_domain(domain)
self.scraper.open_domain(domain)
stats.open_domain(domain)
# XXX: sent for backwards compatibility (will be removed in Scrapy 0.8)
signals.send_catch_log(signals.domain_open, sender=self.__class__, domain=domain, spider=spider)
signals.send_catch_log(signals.domain_opened, sender=self.__class__, domain=domain, spider=spider)
def _domain_idle(self, domain):
@ -325,7 +330,9 @@ class ExecutionEngine(object):
self.scheduler.close_domain(domain)
self.scraper.close_domain(domain)
reason = self.closing.pop(domain, 'finished')
signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, domain=domain, spider=spider, reason=reason)
signals.send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
domain=domain, spider=spider, reason=reason)
stats.close_domain(domain, reason=reason)
log.msg("Domain closed (%s)" % reason, domain=domain)
self._mainloop()

4
scrapy/core/signals.py
View File

@ -11,7 +11,6 @@ from scrapy import log
engine_started = object()
engine_stopped = object()
domain_open = object()
domain_opened = object()
domain_idle = object()
domain_closed = object()
@ -23,6 +22,9 @@ item_scraped = object()
item_passed = object()
item_dropped = object()
# XXX: deprecated signals (will be removed in Scrapy 0.8)
domain_open = object()
def send_catch_log(signal, sender=None, **kwargs):
"""
Send a signal and log any exceptions raised by its listeners

4
scrapy/stats/collector/__init__.py
View File

@ -17,10 +17,6 @@ class StatsCollector(object):
self._dump = settings.getbool('STATS_DUMP')
self._stats = {None: {}} # None is for global stats
dispatcher.connect(self.open_domain, signal=signals.domain_open)
dispatcher.connect(self._start_closing_domain, signal=signals.domain_closed)
dispatcher.connect(self.engine_stopped, signal=signals.engine_stopped)
def get_value(self, key, default=None, domain=None):
return self._stats[domain].get(key, default)