From 13c099a104f433903d5f9ad9469a6de2fcd56062 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 14 Feb 2014 15:16:36 -0200 Subject: [PATCH] fix a reference to unexistent engine.slots. closes #593 --- docs/topics/telnetconsole.rst | 27 ++++++++++++-------------- scrapy/tests/spiders.py | 6 ++++++ scrapy/tests/test_crawl.py | 15 +++++++++++++++ scrapy/utils/engine.py | 36 ++++++++++++----------------------- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index bcfdb09cd..7677ec462 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -83,24 +83,21 @@ using the telnet console:: >>> est() Execution engine status - time()-engine.start_time : 9.24237799644 + time()-engine.start_time : 8.62972998619 engine.has_capacity() : False - engine.downloader.is_idle() : False - len(engine.downloader.slots) : 2 len(engine.downloader.active) : 16 engine.scraper.is_idle() : False - - Spider: - engine.spider_is_idle(spider) : False - engine.slots[spider].closing : False - len(engine.slots[spider].inprogress) : 21 - len(engine.slots[spider].scheduler.dqs or []) : 0 - len(engine.slots[spider].scheduler.mqs) : 4453 - len(engine.scraper.slot.queue) : 0 - len(engine.scraper.slot.active) : 5 - engine.scraper.slot.active_size : 1515069 - engine.scraper.slot.itemproc_size : 0 - engine.scraper.slot.needs_backout() : False + engine.spider.name : followall + engine.spider_is_idle(engine.spider) : False + engine.slot.closing : False + len(engine.slot.inprogress) : 16 + len(engine.slot.scheduler.dqs or []) : 0 + len(engine.slot.scheduler.mqs) : 92 + len(engine.scraper.slot.queue) : 0 + len(engine.scraper.slot.active) : 0 + engine.scraper.slot.active_size : 0 + engine.scraper.slot.itemproc_size : 0 + engine.scraper.slot.needs_backout() : False Pause, resume and stop the Scrapy engine diff --git a/scrapy/tests/spiders.py b/scrapy/tests/spiders.py index 03649ddb9..5cf15818d 100644 --- a/scrapy/tests/spiders.py +++ b/scrapy/tests/spiders.py @@ -137,6 +137,8 @@ class BrokenStartRequestsSpider(FollowAllSpider): class SingleRequestSpider(MetaSpider): seed = None + callback_func = None + errback_func = None def start_requests(self): if isinstance(self.seed, Request): @@ -146,8 +148,12 @@ class SingleRequestSpider(MetaSpider): def parse(self, response): self.meta.setdefault('responses', []).append(response) + if callable(self.callback_func): + return self.callback_func(response) if 'next' in response.meta: return response.meta['next'] def on_error(self, failure): self.meta['failure'] = failure + if callable(self.errback_func): + return self.errback_func(failure) diff --git a/scrapy/tests/test_crawl.py b/scrapy/tests/test_crawl.py index ae068b79c..e6edb4dda 100644 --- a/scrapy/tests/test_crawl.py +++ b/scrapy/tests/test_crawl.py @@ -192,3 +192,18 @@ with multiples lines # last request explicitly sets a Referer header echo3 = json.loads(spider.meta['responses'][3].body) self.assertEqual(echo3['headers'].get('Referer'), ['http://example.com']) + + @defer.inlineCallbacks + def test_engine_status(self): + from scrapy.utils.engine import get_engine_status + est = [] + + def cb(response): + est.append(get_engine_status(spider.crawler.engine)) + + spider = SingleRequestSpider(seed='http://localhost:8998/', callback_func=cb) + yield docrawl(spider) + self.assertEqual(len(est), 1, est) + s = dict(est[0]) + self.assertEqual(s['engine.spider.name'], spider.name) + self.assertEqual(s['len(engine.scraper.slot.active)'], 1) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 3bad4a541..11dd36d91 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -5,14 +5,13 @@ from time import time # used in global tests code def get_engine_status(engine): """Return a report of the current engine status""" - global_tests = [ + tests = [ "time()-engine.start_time", "engine.has_capacity()", "len(engine.downloader.active)", "engine.scraper.is_idle()", - ] - spider_tests = [ - "engine.spider_is_idle(spider)", + "engine.spider.name", + "engine.spider_is_idle(engine.spider)", "engine.slot.closing", "len(engine.slot.inprogress)", "len(engine.slot.scheduler.dqs or [])", @@ -24,34 +23,23 @@ def get_engine_status(engine): "engine.scraper.slot.needs_backout()", ] - status = {'global': [], 'spiders': {}} - for test in global_tests: + checks = [] + for test in tests: try: - status['global'] += [(test, eval(test))] + checks += [(test, eval(test))] except Exception as e: - status['global'] += [(test, "%s (exception)" % type(e).__name__)] - for spider in engine.slots.keys(): - x = [] - for test in spider_tests: - try: - x += [(test, eval(test))] - except Exception as e: - x += [(test, "%s (exception)" % type(e).__name__)] - status['spiders'][spider] = x - return status + checks += [(test, "%s (exception)" % type(e).__name__)] + + return checks def format_engine_status(engine=None): - status = get_engine_status(engine) + checks = get_engine_status(engine) s = "Execution engine status\n\n" - for test, result in status['global']: + for test, result in checks: s += "%-47s : %s\n" % (test, result) s += "\n" - for spider, tests in status['spiders'].items(): - s += "Spider: %s\n" % spider - for test, result in tests: - s += " %-50s : %s\n" % (test, result) + return s def print_engine_status(engine): print(format_engine_status(engine)) -