RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

downloader: renamed SpiderInfo to Slot, for consistency with engine and scraper names

This commit is contained in:
Pablo Hoffman 2011-07-22 02:06:10 -03:00
parent d6b83fee3e
commit 2ac08a713d
4 changed files with 64 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
docs/topics/telnetconsole.rst
View File

@ -79,30 +79,29 @@ using the telnet console::
>>> est()
Execution engine status
datetime.now()-self.start_time : 0:00:09.051588
self.is_idle() : False
self.scheduler.is_idle() : False
len(self.scheduler.pending_requests) : 1
self.downloader.is_idle() : False
len(self.downloader.sites) : 1
self.downloader.has_capacity() : True
self.pipeline.is_idle() : False
len(self.pipeline.domaininfo) : 1
len(self._scraping) : 1
time()-engine.start_time : 21.3188259602
engine.is_idle() : False
engine.has_capacity() : True
engine.scheduler.is_idle() : False
len(engine.scheduler.pending_requests) : 1
engine.downloader.is_idle() : False
len(engine.downloader.slots) : 1
engine.scraper.is_idle() : False
len(engine.scraper.slots) : 1
example.com
self.domain_is_idle(domain) : False
self.closing.get(domain) : None
self.scheduler.domain_has_pending_requests(domain) : True
len(self.scheduler.pending_requests[domain]) : 97
len(self.downloader.sites[domain].queue) : 17
len(self.downloader.sites[domain].active) : 25
len(self.downloader.sites[domain].transferring) : 8
self.downloader.sites[domain].closing : False
self.downloader.sites[domain].lastseen : 2009-06-23 15:20:16.563675
self.pipeline.domain_is_idle(domain) : True
len(self.pipeline.domaininfo[domain]) : 0
len(self._scraping[domain]) : 0
Spider: <GayotSpider 'gayotcom' at 0x2dc2b10>
engine.spider_is_idle(spider) : False
engine.slots[spider].closing : False
len(engine.scheduler.pending_requests[spider]) : 11504
len(engine.downloader.slots[spider].queue) : 9
len(engine.downloader.slots[spider].active) : 17
len(engine.downloader.slots[spider].transferring) : 8
engine.downloader.slots[spider].lastseen : 1311311093.61
len(engine.scraper.slots[spider].queue) : 0
len(engine.scraper.slots[spider].active) : 0
engine.scraper.slots[spider].active_size : 0
engine.scraper.slots[spider].itemproc_size : 0
engine.scraper.slots[spider].needs_backout() : False
Pause, resume and stop the Scrapy engine

60
scrapy/core/downloader/__init__.py
View File

@ -18,8 +18,8 @@ from scrapy import log
from .middleware import DownloaderMiddlewareManager
from .handlers import DownloadHandlers
class SpiderInfo(object):
"""Simple class to keep information and state for each open spider"""
class Slot(object):
"""Downloader spider slot"""
def __init__(self, spider):
setattr_default(spider, 'download_delay', spider.settings.getfloat('DOWNLOAD_DELAY'))
@ -62,7 +62,7 @@ class Downloader(object):
"""
def __init__(self):
self.sites = {}
self.slots = {}
self.handlers = DownloadHandlers()
self.middleware = DownloaderMiddlewareManager.from_settings(settings)
self.concurrent_spiders = settings.getint('CONCURRENT_SPIDERS')
@ -74,11 +74,11 @@ class Downloader(object):
Response object, then request never reach downloader queue, and it will
not be downloaded from site.
"""
site = self.sites[spider]
slot = self.slots[spider]
site.active.add(request)
slot.active.add(request)
def _deactivate(response):
site.active.remove(request)
slot.active.remove(request)
return response
dfd = self.middleware.download(self.enqueue, request, spider)
@ -86,7 +86,7 @@ class Downloader(object):
def enqueue(self, request, spider):
"""Enqueue a Request for a effective download from site"""
site = self.sites[spider]
slot = self.slots[spider]
def _downloaded(response):
send_catch_log(signal=signals.response_downloaded, \
@ -94,37 +94,37 @@ class Downloader(object):
return response
deferred = defer.Deferred().addCallback(_downloaded)
site.queue.append((request, deferred))
slot.queue.append((request, deferred))
self._process_queue(spider)
return deferred
def _process_queue(self, spider):
"""Effective download requests from site queue"""
site = self.sites.get(spider)
if not site:
"""Effective download requests from slot queue"""
slot = self.slots.get(spider)
if not slot:
return
# Delay queue processing if a download_delay is configured
now = time()
delay = site.download_delay()
delay = slot.download_delay()
if delay:
penalty = delay - now + site.lastseen
if penalty > 0 and site.free_transfer_slots():
penalty = delay - now + slot.lastseen
if penalty > 0 and slot.free_transfer_slots():
d = defer.Deferred()
d.addCallback(self._process_queue)
call = reactor.callLater(penalty, d.callback, spider)
site.next_request_calls.add(call)
d.addBoth(lambda x: site.next_request_calls.remove(call))
slot.next_request_calls.add(call)
d.addBoth(lambda x: slot.next_request_calls.remove(call))
return
site.lastseen = now
slot.lastseen = now
# Process enqueued requests if there are free slots to transfer for this site
while site.queue and site.free_transfer_slots() > 0:
request, deferred = site.queue.popleft()
dfd = self._download(site, request, spider)
# Process enqueued requests if there are free slots to transfer for this slot
while slot.queue and slot.free_transfer_slots() > 0:
request, deferred = slot.queue.popleft()
dfd = self._download(slot, request, spider)
dfd.chainDeferred(deferred)
def _download(self, site, request, spider):
def _download(self, slot, request, spider):
# The order is very important for the following deferreds. Do not change!
# 1. Create the download deferred
@ -134,24 +134,24 @@ class Downloader(object):
# state to free up the transferring slot so it can be used by the
# following requests (perhaps those which came from the downloader
# middleware itself)
site.transferring.add(request)
slot.transferring.add(request)
def finish_transferring(_):
site.transferring.remove(request)
slot.transferring.remove(request)
self._process_queue(spider)
return _
return dfd.addBoth(finish_transferring)
def open_spider(self, spider):
"""Allocate resources to begin processing a spider"""
assert spider not in self.sites, "Spider already opened: %s" % spider
self.sites[spider] = SpiderInfo(spider)
assert spider not in self.slots, "Spider already opened: %s" % spider
self.slots[spider] = Slot(spider)
def close_spider(self, spider):
"""Free any resources associated with the given spider"""
assert spider in self.sites, "Spider not opened: %s" % spider
site = self.sites.pop(spider)
site.cancel_request_calls()
assert spider in self.slots, "Spider not opened: %s" % spider
slot = self.slots.pop(spider)
slot.cancel_request_calls()
def is_idle(self):
return not self.sites
return not self.slots

12
scrapy/core/engine.py
View File

@ -124,7 +124,7 @@ class ExecutionEngine(object):
return not self.running \
or slot.closing \
or self.spider_is_closed(spider) \
or self.downloader.sites[spider].needs_backout() \
or self.downloader.slots[spider].needs_backout() \
or self.scraper.slots[spider].needs_backout()
def _next_request(self, spider):
@ -160,22 +160,22 @@ class ExecutionEngine(object):
scraper_idle = spider in self.scraper.slots \
and self.scraper.slots[spider].is_idle()
pending = self.scheduler.spider_has_pending_requests(spider)
downloading = spider in self.downloader.sites \
and self.downloader.sites[spider].active
downloading = spider in self.downloader.slots \
and self.downloader.slots[spider].active
return scraper_idle and not (pending or downloading)
def spider_is_closed(self, spider):
"""Return True if the spider is fully closed (ie. not even in the
closing stage)"""
return spider not in self.downloader.sites
return spider not in self.downloader.slots
@property
def open_spiders(self):
return self.downloader.sites.keys()
return self.downloader.slots.keys()
def has_capacity(self):
"""Does the engine have capacity to handle more spiders"""
return len(self.downloader.sites) < self.downloader.concurrent_spiders
return len(self.downloader.slots) < self.downloader.concurrent_spiders
def crawl(self, request, spider):
assert spider in self.open_spiders, \

12
scrapy/utils/engine.py
View File

@ -16,7 +16,7 @@ def get_engine_status(engine=None):
"engine.scheduler.is_idle()",
"len(engine.scheduler.pending_requests)",
"engine.downloader.is_idle()",
"len(engine.downloader.sites)",
"len(engine.downloader.slots)",
"engine.scraper.is_idle()",
"len(engine.scraper.slots)",
]
@ -24,10 +24,10 @@ def get_engine_status(engine=None):
"engine.spider_is_idle(spider)",
"engine.slots[spider].closing",
"len(engine.scheduler.pending_requests[spider])",
"len(engine.downloader.sites[spider].queue)",
"len(engine.downloader.sites[spider].active)",
"len(engine.downloader.sites[spider].transferring)",
"engine.downloader.sites[spider].lastseen",
"len(engine.downloader.slots[spider].queue)",
"len(engine.downloader.slots[spider].active)",
"len(engine.downloader.slots[spider].transferring)",
"engine.downloader.slots[spider].lastseen",
"len(engine.scraper.slots[spider].queue)",
"len(engine.scraper.slots[spider].active)",
"engine.scraper.slots[spider].active_size",
@ -41,7 +41,7 @@ def get_engine_status(engine=None):
status['global'] += [(test, eval(test))]
except Exception, e:
status['global'] += [(test, "%s (exception)" % type(e).__name__)]
for spider in set(engine.downloader.sites.keys() + engine.scraper.slots.keys()):
for spider in set(engine.downloader.slots.keys() + engine.scraper.slots.keys()):
x = []
for test in spider_tests:
try: