From 2b212d426668c02704bc23c4e988fa108e0738ef Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:06 +0000 Subject: [PATCH 1/6] ignore cache for pytests --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 406146e5f..ff6e2ea65 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ dist .idea htmlcov/ .coverage +.pytest_cache/ .coverage.* .cache/ From d95762db7c9abe59d8edfc8c17397a5a5bbfc661 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:24 +0000 Subject: [PATCH 2/6] new signal --- scrapy/signals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/signals.py b/scrapy/signals.py index e36c27203..c0e4bb74e 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -13,6 +13,7 @@ spider_closed = object() spider_error = object() request_scheduled = object() request_dropped = object() +request_reached_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() From 597b8a97ad468123432dba3d6f2c3dd943f4aa36 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:42 +0000 Subject: [PATCH 3/6] documentation for new signal --- docs/topics/signals.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d40c0e1df..cf7b8db2f 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -279,6 +279,22 @@ request_dropped :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_reached_downloader +--------------------------- + +.. signal:: request_reached_downloader +.. function:: request_reached_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request`, reached downloader. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- From afb1458bd3b75fad78f053ac062910052971cdfc Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:39:54 +0000 Subject: [PATCH 4/6] tests for new signal --- tests/test_engine.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_engine.py b/tests/test_engine.py index 719c0c60c..856465161 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -103,6 +103,7 @@ class CrawlerRun(object): self.respplug = [] self.reqplug = [] self.reqdropped = [] + self.reqreached = [] self.itemerror = [] self.itemresp = [] self.signals_catched = {} @@ -124,6 +125,7 @@ class CrawlerRun(object): self.crawler.signals.connect(self.item_error, signals.item_error) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) + self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) self.crawler.crawl(start_urls=start_urls) self.spider = self.crawler.spider @@ -155,6 +157,9 @@ class CrawlerRun(object): def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) + def request_reached(self, request, spider): + self.reqreached.append((request, spider)) + def request_dropped(self, request, spider): self.reqdropped.append((request, spider)) @@ -212,6 +217,8 @@ class EngineTest(unittest.TestCase): responses_count = len(self.run.respplug) self.assertEqual(scheduled_requests_count, dropped_requests_count + responses_count) + self.assertEqual(len(self.run.reqreached), + responses_count) def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) @@ -219,6 +226,7 @@ class EngineTest(unittest.TestCase): def _assert_downloaded_responses(self): # response tests self.assertEqual(8, len(self.run.respplug)) + self.assertEqual(8, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': From 561ad3b63c539b560bbaed1df37d0b60043ab3b3 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 17 Aug 2018 14:40:24 +0000 Subject: [PATCH 5/6] emit new signal --- scrapy/core/downloader/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d835e65f7..59c3ad074 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -129,6 +129,9 @@ class Downloader(object): return response slot.active.add(request) + self.signals.send_catch_log(signal=signals.request_reached_downloader, + request=request, + spider=spider) deferred = defer.Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) From c02cfa574cc47d6b086cc66025ddef9f3174ac02 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 29 Aug 2018 11:21:55 +0000 Subject: [PATCH 6/6] remove comma --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index cf7b8db2f..ff07b9d55 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -285,7 +285,7 @@ request_reached_downloader .. signal:: request_reached_downloader .. function:: request_reached_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request`, reached downloader. + Sent when a :class:`~scrapy.http.Request` reached downloader. The signal does not support returning deferreds from their handlers.