diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index c93a48597..9cb659e67 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -209,6 +209,7 @@ class ScrapyAgent(object): self._pool = pool self._maxsize = maxsize self._warnsize = warnsize + self._txresponse = None def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress @@ -275,6 +276,11 @@ class ScrapyAgent(object): if self._timeout_cl.active(): self._timeout_cl.cancel() return result + # needed for HTTPS requests, otherwise _ResponseReader doesn't + # receive connectionLost() + if self._txresponse: + self._txresponse._transport.stopProducing() + raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout)) def _cb_latency(self, result, request, start_time): @@ -310,6 +316,10 @@ class ScrapyAgent(object): d = defer.Deferred(_cancel) txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize)) + + # save response for timeouts + self._txresponse = txresponse + return d def _cb_bodydone(self, result, request, url): diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 9bcc51943..1c89a0f9e 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -83,6 +83,11 @@ class ScrapyHTTPPageGetter(HTTPClient): def timeout(self): self.transport.loseConnection() + + # transport cleanup needed for HTTPS connections + if self.factory.url.startswith(b'https'): + self.transport.stopProducing() + self.factory.noPage(\ defer.TimeoutError("Getting %s took longer than %s seconds." % \ (self.factory.url, self.factory.timeout))) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 88b60e4ff..e13becf1b 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -182,17 +182,19 @@ class HttpTestCase(unittest.TestCase): return d @defer.inlineCallbacks - def test_timeout_download_from_spider(self): - if self.scheme == 'https': - raise unittest.SkipTest( - 'test_timeout_download_from_spider skipped under https') + def test_timeout_download_from_spider_nodata_rcvd(self): + # client connects but no data is received spider = Spider('foo') meta = {'download_timeout': 0.2} - # client connects but no data is received request = Request(self.getURL('wait'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError) + + @defer.inlineCallbacks + def test_timeout_download_from_spider_server_hangs(self): # client connects, server send headers and some body bytes but hangs + spider = Spider('foo') + meta = {'download_timeout': 0.2} request = Request(self.getURL('hang-after-headers'), meta=meta) d = self.download_request(request, spider) yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)