From 3b1db71dac8716878ff1b94ee0d1095e5c80795f Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 9 Jan 2019 12:14:40 +0000 Subject: [PATCH] New signal update signature documentation for new signal utilize new signal correct signal handler signature emit new signal test another signal new signal rename test file faster test rename test case tests for signal emitting in bad cases --- docs/topics/signals.rst | 17 +++++++++ scrapy/core/downloader/__init__.py | 3 ++ scrapy/pqueues.py | 6 +-- scrapy/signals.py | 1 + tests/test_request_left.py | 59 ++++++++++++++++++++++++++++++ tests/test_scheduler.py | 8 ++-- 6 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 tests/test_request_left.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ff07b9d55..f13e8270c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,6 +295,23 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_left_downloader +--------------------------- + +.. signal:: request_left_downloader +.. function:: request_left_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request` left downloader even in case of + failure. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4695d75f4..d856a2f37 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -188,6 +188,9 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) + self.signals.send_catch_log(signal=signals.request_left_downloader, + request=request, + spider=spider) return _ return dfd.addBoth(finish_transferring) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index d8eed010f..6a9feb599 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -7,7 +7,7 @@ from queuelib import PriorityQueue from scrapy.core.downloader import Downloader from scrapy.http import Request -from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.utils.httpobj import urlparse_cached @@ -163,7 +163,7 @@ class DownloaderAwarePriorityQueue(object): self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, - signal=response_downloaded) + signal=request_left_downloader) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) @@ -199,7 +199,7 @@ class DownloaderAwarePriorityQueue(object): if slot not in self._active_downloads: self._active_downloads[slot] = 0 - def on_response_download(self, response, request, spider): + def on_response_download(self, request, spider): if not self.check_mark(request): return self.unmark(request) diff --git a/scrapy/signals.py b/scrapy/signals.py index c0e4bb74e..2ea986b8c 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,6 +14,7 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() +request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() diff --git a/tests/test_request_left.py b/tests/test_request_left.py new file mode 100644 index 000000000..ddeca0499 --- /dev/null +++ b/tests/test_request_left.py @@ -0,0 +1,59 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.signals import request_left_downloader +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + +class SignalCatcherSpider(Spider): + name = 'signal_catcher' + + def __init__(self, crawler, url, *args, **kwargs): + super(SignalCatcherSpider, self).__init__(*args, **kwargs) + crawler.signals.connect(self.on_response_download, + signal=request_left_downloader) + self.catched_times = 0 + self.start_urls = [url] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(crawler, *args, **kwargs) + return spider + + def on_response_download(self, request, spider): + self.catched_times = self.catched_times + 1 + + +class TestCatching(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_success(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/status?n=200")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_timeout(self): + crawler = get_crawler(SignalCatcherSpider, + {'DOWNLOAD_TIMEOUT': 0.1}) + yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_disconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/drop")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_noconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') + self.assertEqual(crawler.spider.catched_times, 1) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 3fb70a110..1bcc1e5a8 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -9,7 +9,7 @@ from scrapy.crawler import Crawler from scrapy.core.scheduler import Scheduler from scrapy.http import Request from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write -from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -216,9 +216,8 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, for request in requests: self.mock_crawler.signals.send_catch_log( - signal=response_downloaded, + signal=request_left_downloader, request=request, - response=None, spider=self.spider ) @@ -265,9 +264,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, for request in requests: self.mock_crawler.signals.send_catch_log( - signal=response_downloaded, + signal=request_left_downloader, request=request, - response=None, spider=self.spider )