RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed "No free spider slots" bug when calling fetch() from scrapy shell

This commit is contained in:
Pablo Hoffman 2011-11-14 20:03:43 -02:00
parent d37a788d22
commit 6d13de4366
1 changed files with 12 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrapy/shell.py
View File

@ -29,6 +29,7 @@ class Shell(object):
self.crawler = crawler
self.update_vars = update_vars or (lambda x: None)
self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS'])
self.spider = None
self.inthread = inthread
self.code = code
self.vars = {}
@ -57,15 +58,22 @@ class Shell(object):
start_python_console(self.vars)
def _schedule(self, request, spider):
spider = self._open_spider(request, spider)
d = request_deferred(request)
d.addCallback(lambda x: (x, spider))
self.crawler.engine.crawl(request, spider)
return d
def _open_spider(self, request, spider):
if self.spider:
return self.spider
if spider is None:
spider = create_spider_for_request(self.crawler.spiders, request, \
BaseSpider('default'), log_multiple=True)
spider.set_crawler(self.crawler)
self.crawler.engine.open_spider(spider, close_if_idle=False)
d = request_deferred(request)
d.addCallback(lambda x: (x, spider))
self.crawler.engine.crawl(request, spider)
return d
self.spider = spider
return spider
def fetch(self, request_or_url, spider=None):
if isinstance(request_or_url, Request):