From 898bc00811aac9d3e38d1863b95a10c2e8effb02 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:31:27 +0000 Subject: [PATCH 01/12] new signal --- scrapy/signals.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/signals.py b/scrapy/signals.py index 6b9125302..cd7ed7fb1 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,6 +14,7 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() +request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() From ae04174884eeb777d7b3caceed52bf522944ceb1 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:32:31 +0000 Subject: [PATCH 02/12] emit new signal --- scrapy/core/downloader/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 157dc3418..5a2fdadf5 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -181,6 +181,9 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) + self.signals.send_catch_log(signal=signals.request_left_downloader, + request=request, + spider=spider) return _ return dfd.addBoth(finish_transferring) From 9916f6e556f9d4a41ea86d4a73687af1a40e43ba Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:32:54 +0000 Subject: [PATCH 03/12] tests for new signal --- tests/test_request_left.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/test_request_left.py diff --git a/tests/test_request_left.py b/tests/test_request_left.py new file mode 100644 index 000000000..ddeca0499 --- /dev/null +++ b/tests/test_request_left.py @@ -0,0 +1,59 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.signals import request_left_downloader +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + +class SignalCatcherSpider(Spider): + name = 'signal_catcher' + + def __init__(self, crawler, url, *args, **kwargs): + super(SignalCatcherSpider, self).__init__(*args, **kwargs) + crawler.signals.connect(self.on_response_download, + signal=request_left_downloader) + self.catched_times = 0 + self.start_urls = [url] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(crawler, *args, **kwargs) + return spider + + def on_response_download(self, request, spider): + self.catched_times = self.catched_times + 1 + + +class TestCatching(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_success(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/status?n=200")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_timeout(self): + crawler = get_crawler(SignalCatcherSpider, + {'DOWNLOAD_TIMEOUT': 0.1}) + yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_disconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/drop")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_noconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') + self.assertEqual(crawler.spider.catched_times, 1) From aab39f63412b4b7a0ae2713446859d6d8103e5f7 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 11:35:03 +0000 Subject: [PATCH 04/12] docummentation for new signal --- docs/topics/signals.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3f29aa323..7fa5bc030 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,6 +295,23 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_left_downloader +--------------------------- + +.. signal:: request_left_downloader +.. function:: request_left_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request` left downloader even in case of + failure. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- From 3769f75386104c1a3072894b302d3c3239ff8c37 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Feb 2020 12:08:08 +0000 Subject: [PATCH 05/12] pep8 E302 --- tests/test_request_left.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index ddeca0499..5d271190d 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -5,6 +5,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer + class SignalCatcherSpider(Spider): name = 'signal_catcher' From 6733f4d976150e0e5352d4ae9697880ae60ad638 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 6 Feb 2020 18:40:42 +0500 Subject: [PATCH 06/12] Update docs/topics/signals.rst Co-Authored-By: elacuesta --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 7fa5bc030..47be6b603 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -301,7 +301,7 @@ request_left_downloader .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` left downloader even in case of + Sent when a :class:`~scrapy.http.Request` leaves the downloader even in case of failure. The signal does not support returning deferreds from their handlers. From 4a91a5427df4846ed9fa11612cfeb9e31f34a1c8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 6 Feb 2020 13:44:51 +0000 Subject: [PATCH 07/12] fix typo --- tests/test_request_left.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 5d271190d..8256d1c92 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -13,7 +13,7 @@ class SignalCatcherSpider(Spider): super(SignalCatcherSpider, self).__init__(*args, **kwargs) crawler.signals.connect(self.on_response_download, signal=request_left_downloader) - self.catched_times = 0 + self.caught_times = 0 self.start_urls = [url] @classmethod @@ -22,7 +22,7 @@ class SignalCatcherSpider(Spider): return spider def on_response_download(self, request, spider): - self.catched_times = self.catched_times + 1 + self.caught_times = self.caught_times + 1 class TestCatching(TestCase): @@ -38,23 +38,23 @@ class TestCatching(TestCase): def test_success(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/status?n=200")) - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) @defer.inlineCallbacks def test_timeout(self): crawler = get_crawler(SignalCatcherSpider, {'DOWNLOAD_TIMEOUT': 0.1}) yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) @defer.inlineCallbacks def test_disconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl(self.mockserver.url("/drop")) - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) @defer.inlineCallbacks def test_noconnect(self): crawler = get_crawler(SignalCatcherSpider) yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') - self.assertEqual(crawler.spider.catched_times, 1) + self.assertEqual(crawler.spider.caught_times, 1) From 4be19e443e9c101a248c21509ae8000ce500d51a Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 6 Feb 2020 13:46:23 +0000 Subject: [PATCH 08/12] name signla catcher in accord with signal name --- tests/test_request_left.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 8256d1c92..5cfef8e7d 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -11,7 +11,7 @@ class SignalCatcherSpider(Spider): def __init__(self, crawler, url, *args, **kwargs): super(SignalCatcherSpider, self).__init__(*args, **kwargs) - crawler.signals.connect(self.on_response_download, + crawler.signals.connect(self.on_request_left, signal=request_left_downloader) self.caught_times = 0 self.start_urls = [url] @@ -21,7 +21,7 @@ class SignalCatcherSpider(Spider): spider = cls(crawler, *args, **kwargs) return spider - def on_response_download(self, request, spider): + def on_request_left(self, request, spider): self.caught_times = self.caught_times + 1 From 84b55b73646acca71461366ef98a1a501331f5d8 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:07:35 +0500 Subject: [PATCH 09/12] Update docs/topics/signals.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 47be6b603..60d9ce2bc 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -306,7 +306,7 @@ request_left_downloader The signal does not support returning deferreds from their handlers. - :param request: the request that reached downloader + :param request: the request that reached the downloader :type request: :class:`~scrapy.http.Request` object :param spider: the spider that yielded the request From 2f83f3e2cb3497e89d42533b8f20f8398a696f46 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:07:43 +0500 Subject: [PATCH 10/12] Update docs/topics/signals.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 60d9ce2bc..49475c1af 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -301,7 +301,7 @@ request_left_downloader .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) - Sent when a :class:`~scrapy.http.Request` leaves the downloader even in case of + Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of failure. The signal does not support returning deferreds from their handlers. From 8817b9e8e92f01147e7e44dd767165766093f408 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:07:53 +0500 Subject: [PATCH 11/12] Update docs/topics/signals.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Adrián Chaves --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 49475c1af..a7d60e9cb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -296,7 +296,7 @@ request_reached_downloader :type spider: :class:`~scrapy.spiders.Spider` object request_left_downloader ---------------------------- +----------------------- .. signal:: request_left_downloader .. function:: request_left_downloader(request, spider) From 153b78e53f5c0f4630d09d560e111ca68f357905 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 7 Feb 2020 11:08:55 +0500 Subject: [PATCH 12/12] Update docs/topics/signals.rst Co-Authored-By: elacuesta --- docs/topics/signals.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index a7d60e9cb..886d1b866 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -304,7 +304,7 @@ request_left_downloader Sent when a :class:`~scrapy.http.Request` leaves the downloader, even in case of failure. - The signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param request: the request that reached the downloader :type request: :class:`~scrapy.http.Request` object