mirror of https://github.com/scrapy/scrapy.git
reverted an experimental code that should have been commited
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40293
This commit is contained in:
parent
1f4e484a5c
commit
e447044fb9
|
|
@ -35,39 +35,16 @@ class ExecutionManager(object):
|
|||
|
||||
requests = self._parse_args(args)
|
||||
self.priorities = self.prioritizer_class(requests.keys())
|
||||
|
||||
#applies a time (in seconds) between succesive requests crawl
|
||||
#useful for applications that runs softly distributed in time
|
||||
self.crawl_delay = opts.get("crawl_delay", 0)
|
||||
|
||||
|
||||
def crawl(self, *args):
|
||||
"""Schedule the given args for crawling. args is a list of urls or domains"""
|
||||
|
||||
requests = self._parse_args(args)
|
||||
# schedule initial requets to be scraped at engine start
|
||||
|
||||
def _issue():
|
||||
for domain in requests or ():
|
||||
spider = spiders.fromdomain(domain)
|
||||
priority = self.priorities.get_priority(domain)
|
||||
for request in requests[domain]:
|
||||
yield request, spider, priority
|
||||
|
||||
if self.crawl_delay:
|
||||
gen = _issue()
|
||||
def _soft_crawl():
|
||||
try:
|
||||
request, spider, priority = gen.next()
|
||||
scrapyengine.crawl(request, spider, domain_priority=priority)
|
||||
log.msg("Delaying %ss the next request." % self.crawl_delay)
|
||||
except StopIteration:
|
||||
self.stop()
|
||||
|
||||
scrapyengine.addtask(_soft_crawl, self.crawl_delay)
|
||||
|
||||
else:
|
||||
for request, spider, priority in _issue():
|
||||
for domain in requests or ():
|
||||
spider = spiders.fromdomain(domain)
|
||||
priority = self.priorities.get_priority(domain)
|
||||
for request in requests[domain]:
|
||||
scrapyengine.crawl(request, spider, domain_priority=priority)
|
||||
|
||||
def runonce(self, *args, **opts):
|
||||
|
|
|
|||
Loading…
Reference in New Issue