diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 27d170c77..b60eafdd5 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -79,30 +79,29 @@ using the telnet console:: >>> est() Execution engine status - datetime.now()-self.start_time : 0:00:09.051588 - self.is_idle() : False - self.scheduler.is_idle() : False - len(self.scheduler.pending_requests) : 1 - self.downloader.is_idle() : False - len(self.downloader.sites) : 1 - self.downloader.has_capacity() : True - self.pipeline.is_idle() : False - len(self.pipeline.domaininfo) : 1 - len(self._scraping) : 1 + time()-engine.start_time : 21.3188259602 + engine.is_idle() : False + engine.has_capacity() : True + engine.scheduler.is_idle() : False + len(engine.scheduler.pending_requests) : 1 + engine.downloader.is_idle() : False + len(engine.downloader.slots) : 1 + engine.scraper.is_idle() : False + len(engine.scraper.slots) : 1 - example.com - self.domain_is_idle(domain) : False - self.closing.get(domain) : None - self.scheduler.domain_has_pending_requests(domain) : True - len(self.scheduler.pending_requests[domain]) : 97 - len(self.downloader.sites[domain].queue) : 17 - len(self.downloader.sites[domain].active) : 25 - len(self.downloader.sites[domain].transferring) : 8 - self.downloader.sites[domain].closing : False - self.downloader.sites[domain].lastseen : 2009-06-23 15:20:16.563675 - self.pipeline.domain_is_idle(domain) : True - len(self.pipeline.domaininfo[domain]) : 0 - len(self._scraping[domain]) : 0 + Spider: + engine.spider_is_idle(spider) : False + engine.slots[spider].closing : False + len(engine.scheduler.pending_requests[spider]) : 11504 + len(engine.downloader.slots[spider].queue) : 9 + len(engine.downloader.slots[spider].active) : 17 + len(engine.downloader.slots[spider].transferring) : 8 + engine.downloader.slots[spider].lastseen : 1311311093.61 + len(engine.scraper.slots[spider].queue) : 0 + len(engine.scraper.slots[spider].active) : 0 + engine.scraper.slots[spider].active_size : 0 + engine.scraper.slots[spider].itemproc_size : 0 + engine.scraper.slots[spider].needs_backout() : False Pause, resume and stop the Scrapy engine diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 832745e28..5e3069e34 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -18,8 +18,8 @@ from scrapy import log from .middleware import DownloaderMiddlewareManager from .handlers import DownloadHandlers -class SpiderInfo(object): - """Simple class to keep information and state for each open spider""" +class Slot(object): + """Downloader spider slot""" def __init__(self, spider): setattr_default(spider, 'download_delay', spider.settings.getfloat('DOWNLOAD_DELAY')) @@ -62,7 +62,7 @@ class Downloader(object): """ def __init__(self): - self.sites = {} + self.slots = {} self.handlers = DownloadHandlers() self.middleware = DownloaderMiddlewareManager.from_settings(settings) self.concurrent_spiders = settings.getint('CONCURRENT_SPIDERS') @@ -74,11 +74,11 @@ class Downloader(object): Response object, then request never reach downloader queue, and it will not be downloaded from site. """ - site = self.sites[spider] + slot = self.slots[spider] - site.active.add(request) + slot.active.add(request) def _deactivate(response): - site.active.remove(request) + slot.active.remove(request) return response dfd = self.middleware.download(self.enqueue, request, spider) @@ -86,7 +86,7 @@ class Downloader(object): def enqueue(self, request, spider): """Enqueue a Request for a effective download from site""" - site = self.sites[spider] + slot = self.slots[spider] def _downloaded(response): send_catch_log(signal=signals.response_downloaded, \ @@ -94,37 +94,37 @@ class Downloader(object): return response deferred = defer.Deferred().addCallback(_downloaded) - site.queue.append((request, deferred)) + slot.queue.append((request, deferred)) self._process_queue(spider) return deferred def _process_queue(self, spider): - """Effective download requests from site queue""" - site = self.sites.get(spider) - if not site: + """Effective download requests from slot queue""" + slot = self.slots.get(spider) + if not slot: return # Delay queue processing if a download_delay is configured now = time() - delay = site.download_delay() + delay = slot.download_delay() if delay: - penalty = delay - now + site.lastseen - if penalty > 0 and site.free_transfer_slots(): + penalty = delay - now + slot.lastseen + if penalty > 0 and slot.free_transfer_slots(): d = defer.Deferred() d.addCallback(self._process_queue) call = reactor.callLater(penalty, d.callback, spider) - site.next_request_calls.add(call) - d.addBoth(lambda x: site.next_request_calls.remove(call)) + slot.next_request_calls.add(call) + d.addBoth(lambda x: slot.next_request_calls.remove(call)) return - site.lastseen = now + slot.lastseen = now - # Process enqueued requests if there are free slots to transfer for this site - while site.queue and site.free_transfer_slots() > 0: - request, deferred = site.queue.popleft() - dfd = self._download(site, request, spider) + # Process enqueued requests if there are free slots to transfer for this slot + while slot.queue and slot.free_transfer_slots() > 0: + request, deferred = slot.queue.popleft() + dfd = self._download(slot, request, spider) dfd.chainDeferred(deferred) - def _download(self, site, request, spider): + def _download(self, slot, request, spider): # The order is very important for the following deferreds. Do not change! # 1. Create the download deferred @@ -134,24 +134,24 @@ class Downloader(object): # state to free up the transferring slot so it can be used by the # following requests (perhaps those which came from the downloader # middleware itself) - site.transferring.add(request) + slot.transferring.add(request) def finish_transferring(_): - site.transferring.remove(request) + slot.transferring.remove(request) self._process_queue(spider) return _ return dfd.addBoth(finish_transferring) def open_spider(self, spider): """Allocate resources to begin processing a spider""" - assert spider not in self.sites, "Spider already opened: %s" % spider - self.sites[spider] = SpiderInfo(spider) + assert spider not in self.slots, "Spider already opened: %s" % spider + self.slots[spider] = Slot(spider) def close_spider(self, spider): """Free any resources associated with the given spider""" - assert spider in self.sites, "Spider not opened: %s" % spider - site = self.sites.pop(spider) - site.cancel_request_calls() + assert spider in self.slots, "Spider not opened: %s" % spider + slot = self.slots.pop(spider) + slot.cancel_request_calls() def is_idle(self): - return not self.sites + return not self.slots diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 386f07406..464668391 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -124,7 +124,7 @@ class ExecutionEngine(object): return not self.running \ or slot.closing \ or self.spider_is_closed(spider) \ - or self.downloader.sites[spider].needs_backout() \ + or self.downloader.slots[spider].needs_backout() \ or self.scraper.slots[spider].needs_backout() def _next_request(self, spider): @@ -160,22 +160,22 @@ class ExecutionEngine(object): scraper_idle = spider in self.scraper.slots \ and self.scraper.slots[spider].is_idle() pending = self.scheduler.spider_has_pending_requests(spider) - downloading = spider in self.downloader.sites \ - and self.downloader.sites[spider].active + downloading = spider in self.downloader.slots \ + and self.downloader.slots[spider].active return scraper_idle and not (pending or downloading) def spider_is_closed(self, spider): """Return True if the spider is fully closed (ie. not even in the closing stage)""" - return spider not in self.downloader.sites + return spider not in self.downloader.slots @property def open_spiders(self): - return self.downloader.sites.keys() + return self.downloader.slots.keys() def has_capacity(self): """Does the engine have capacity to handle more spiders""" - return len(self.downloader.sites) < self.downloader.concurrent_spiders + return len(self.downloader.slots) < self.downloader.concurrent_spiders def crawl(self, request, spider): assert spider in self.open_spiders, \ diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index f6e65c00f..90cddbefe 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -16,7 +16,7 @@ def get_engine_status(engine=None): "engine.scheduler.is_idle()", "len(engine.scheduler.pending_requests)", "engine.downloader.is_idle()", - "len(engine.downloader.sites)", + "len(engine.downloader.slots)", "engine.scraper.is_idle()", "len(engine.scraper.slots)", ] @@ -24,10 +24,10 @@ def get_engine_status(engine=None): "engine.spider_is_idle(spider)", "engine.slots[spider].closing", "len(engine.scheduler.pending_requests[spider])", - "len(engine.downloader.sites[spider].queue)", - "len(engine.downloader.sites[spider].active)", - "len(engine.downloader.sites[spider].transferring)", - "engine.downloader.sites[spider].lastseen", + "len(engine.downloader.slots[spider].queue)", + "len(engine.downloader.slots[spider].active)", + "len(engine.downloader.slots[spider].transferring)", + "engine.downloader.slots[spider].lastseen", "len(engine.scraper.slots[spider].queue)", "len(engine.scraper.slots[spider].active)", "engine.scraper.slots[spider].active_size", @@ -41,7 +41,7 @@ def get_engine_status(engine=None): status['global'] += [(test, eval(test))] except Exception, e: status['global'] += [(test, "%s (exception)" % type(e).__name__)] - for spider in set(engine.downloader.sites.keys() + engine.scraper.slots.keys()): + for spider in set(engine.downloader.slots.keys() + engine.scraper.slots.keys()): x = [] for test in spider_tests: try: