From bef8ade95639ef30fb335e7f7fc9cd71cdc7b233 Mon Sep 17 00:00:00 2001 From: nramirezuy Date: Wed, 26 Jun 2013 16:43:36 -0300 Subject: [PATCH] removed request_received and added request_scheduled --- docs/topics/signals.rst | 13 +++++++------ scrapy/core/engine.py | 2 ++ scrapy/core/scraper.py | 2 -- scrapy/signals.py | 7 +++++-- scrapy/tests/test_engine.py | 13 ++++++------- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 81e1eb2b6..71dc54955 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -181,20 +181,21 @@ spider_error :type spider: :class:`~scrapy.spider.BaseSpider` object -request_received +request_scheduled ---------------- -.. signal:: request_received -.. function:: request_received(request, spider) +.. signal:: request_scheduled +.. function:: request_scheduled(request, spider) - Sent when the engine receives a :class:`~scrapy.http.Request` from a spider. + Sent when the engine schedule a :class:`~scrapy.http.Request`. + It will receive the requests generated from middlewares. This signal does not support returning deferreds from their handlers. - :param request: the request received + :param request: the request scheduled :type request: :class:`~scrapy.http.Request` object - :param spider: the spider which generated the request + :param spider: the spider which generated the original request :type spider: :class:`~scrapy.spider.BaseSpider` object response_received diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 9f5e06a09..aa66b1013 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -174,6 +174,8 @@ class ExecutionEngine(object): self.slots[spider].nextcall.schedule() def schedule(self, request, spider): + self.signals.send_catch_log(signal=signals.request_scheduled, + request=request, spider=spider) return self.slots[spider].scheduler.enqueue_request(request) def download(self, request, spider): diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e871bad83..70214e043 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -162,8 +162,6 @@ class Scraper(object): from the given spider """ if isinstance(output, Request): - self.signals.send_catch_log(signal=signals.request_received, request=output, \ - spider=spider) self.crawler.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): self.slot.itemproc_size += 1 diff --git a/scrapy/signals.py b/scrapy/signals.py index 79de55a4b..11bbae945 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -11,14 +11,17 @@ spider_opened = object() spider_idle = object() spider_closed = object() spider_error = object() -request_received = object() +request_scheduled = object() response_received = object() response_downloaded = object() item_scraped = object() item_dropped = object() +# for backwards compatibility stats_spider_opened = spider_opened stats_spider_closing = spider_closed stats_spider_closed = spider_closed -item_passed = item_scraped # for backwards compatibility +item_passed = item_scraped + +request_received = request_scheduled diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 5b430fb60..c865a4c90 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -95,7 +95,7 @@ class CrawlerRun(object): self.crawler.install() self.crawler.configure() self.crawler.signals.connect(self.item_scraped, signals.item_scraped) - self.crawler.signals.connect(self.request_received, signals.request_received) + self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) self.crawler.crawl(self.spider) self.crawler.start() @@ -122,7 +122,7 @@ class CrawlerRun(object): def item_scraped(self, item, spider, response): self.itemresp.append((item, response)) - def request_received(self, request, spider): + def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) def response_downloaded(self, response, spider): @@ -143,21 +143,20 @@ class EngineTest(unittest.TestCase): self.run = CrawlerRun() yield self.run.run() self._assert_visited_urls() - self._assert_received_requests() + self._assert_scheduled_requests() self._assert_downloaded_responses() self._assert_scraped_items() self._assert_signals_catched() def _assert_visited_urls(self): - must_be_visited = ["/", "/redirect", "/redirected", + must_be_visited = ["/", "/redirect", "/redirected", "/item1.html", "/item2.html", "/item999.html"] urls_visited = set([rp[0].url for rp in self.run.respplug]) urls_expected = set([self.run.geturl(p) for p in must_be_visited]) assert urls_expected <= urls_visited, "URLs not visited: %s" % list(urls_expected - urls_visited) - def _assert_received_requests(self): - # 3 requests should be received from the spider. start_urls and redirects don't count - self.assertEqual(3, len(self.run.reqplug)) + def _assert_scheduled_requests(self): + self.assertEqual(6, len(self.run.reqplug)) paths_expected = ['/item999.html', '/item2.html', '/item1.html']