From 09ba4ff68a7ecb00036d02ffda9ee48f48b69402 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Tue, 30 Dec 2014 19:53:50 -0300 Subject: [PATCH] Patches Twisted issue while closing the connection pool on HTTPDownloadHandler --- scrapy/core/downloader/handlers/http11.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index dd3ad488b..19db71bd1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -32,6 +32,7 @@ class HTTP11DownloadHandler(object): self._contextFactory = self._contextFactoryClass() self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') + self._disconnect_timeout = 1 def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -41,7 +42,24 @@ class HTTP11DownloadHandler(object): return agent.download_request(request) def close(self): - return self._pool.closeCachedConnections() + d = self._pool.closeCachedConnections() + # closeCachedConnections will hang on network or server issues, so + # we'll manually timeout the deferred. + # + # Twisted issue addressing this problem can be found here: + # https://twistedmatrix.com/trac/ticket/7738. + # + # closeCachedConnections doesn't handle external errbacks, so we'll + # issue a callback after `_disconnect_timeout` seconds. + delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) + + def cancel_delayed_call(result): + if delayed_call.active(): + delayed_call.cancel() + return result + + d.addBoth(cancel_delayed_call) + return d class TunnelError(Exception):