mirror of https://github.com/scrapy/scrapy.git
Revert "new signal"
This reverts commit 646164fd7d6dd52061804d2df7424cff929bf739. remove tests Revert "emit new signal" This reverts commit fcde0c6880678957a76af6083b6248f430a00fcf. Revert "documentation for new signal" This reverts commit 8aeb9f696ece95c16499a96767a7afa3d9c4abf4.
This commit is contained in:
parent
443fb98a47
commit
989bba6cb3
|
|
@ -295,23 +295,6 @@ request_reached_downloader
|
|||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
request_left_downloader
|
||||
---------------------------
|
||||
|
||||
.. signal:: request_left_downloader
|
||||
.. function:: request_left_downloader(request, spider)
|
||||
|
||||
Sent when a :class:`~scrapy.http.Request` left downloader even in case of
|
||||
failure.
|
||||
|
||||
The signal does not support returning deferreds from their handlers.
|
||||
|
||||
:param request: the request that reached downloader
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
||||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
response_received
|
||||
-----------------
|
||||
|
||||
|
|
|
|||
|
|
@ -188,9 +188,6 @@ class Downloader(object):
|
|||
def finish_transferring(_):
|
||||
slot.transferring.remove(request)
|
||||
self._process_queue(spider, slot)
|
||||
self.signals.send_catch_log(signal=signals.request_left_downloader,
|
||||
request=request,
|
||||
spider=spider)
|
||||
return _
|
||||
|
||||
return dfd.addBoth(finish_transferring)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ spider_error = object()
|
|||
request_scheduled = object()
|
||||
request_dropped = object()
|
||||
request_reached_downloader = object()
|
||||
request_left_downloader = object()
|
||||
response_received = object()
|
||||
response_downloaded = object()
|
||||
item_scraped = object()
|
||||
|
|
|
|||
|
|
@ -1,59 +0,0 @@
|
|||
from twisted.internet import defer
|
||||
from twisted.trial.unittest import TestCase
|
||||
from scrapy.signals import request_left_downloader
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
class SignalCatcherSpider(Spider):
|
||||
name = 'signal_catcher'
|
||||
|
||||
def __init__(self, crawler, url, *args, **kwargs):
|
||||
super(SignalCatcherSpider, self).__init__(*args, **kwargs)
|
||||
crawler.signals.connect(self.on_response_download,
|
||||
signal=request_left_downloader)
|
||||
self.catched_times = 0
|
||||
self.start_urls = [url]
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = cls(crawler, *args, **kwargs)
|
||||
return spider
|
||||
|
||||
def on_response_download(self, request, spider):
|
||||
self.catched_times = self.catched_times + 1
|
||||
|
||||
|
||||
class TestCatching(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mockserver = MockServer()
|
||||
self.mockserver.__enter__()
|
||||
|
||||
def tearDown(self):
|
||||
self.mockserver.__exit__(None, None, None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_success(self):
|
||||
crawler = get_crawler(SignalCatcherSpider)
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200"))
|
||||
self.assertEqual(crawler.spider.catched_times, 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_timeout(self):
|
||||
crawler = get_crawler(SignalCatcherSpider,
|
||||
{'DOWNLOAD_TIMEOUT': 0.1})
|
||||
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
|
||||
self.assertEqual(crawler.spider.catched_times, 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_disconnect(self):
|
||||
crawler = get_crawler(SignalCatcherSpider)
|
||||
yield crawler.crawl(self.mockserver.url("/drop"))
|
||||
self.assertEqual(crawler.spider.catched_times, 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_noconnect(self):
|
||||
crawler = get_crawler(SignalCatcherSpider)
|
||||
yield crawler.crawl('http://thereisdefinetelynosuchdomain.com')
|
||||
self.assertEqual(crawler.spider.catched_times, 1)
|
||||
Loading…
Reference in New Issue