diff --git a/scrapy/trunk/scrapy/patches/monkeypatches.py b/scrapy/trunk/scrapy/patches/monkeypatches.py index bad1edebd..ce600d774 100644 --- a/scrapy/trunk/scrapy/patches/monkeypatches.py +++ b/scrapy/trunk/scrapy/patches/monkeypatches.py @@ -3,7 +3,8 @@ Monkey patches These are generally a bad idea. """ -from twisted.web.client import HTTPClientFactory +from twisted.web.client import HTTPClientFactory, HTTPPageGetter +import twisted # Extend limit for BeautifulSoup parsing loops @@ -13,6 +14,9 @@ from twisted.web.client import HTTPClientFactory def apply_patches(): patch_HTTPClientFactory_gotHeaders() + if twisted.__version__ < '8.0.0': + patch_HTTPPageGetter_handleResponse() + # XXX: Monkeypatch for twisted.web.client-HTTPClientFactory # HTTPClientFactory.gotHeaders dies when parsing malformed cookies, @@ -43,3 +47,27 @@ def _new_gotHeaders(self, headers): def patch_HTTPClientFactory_gotHeaders(): setattr(HTTPClientFactory, 'gotHeaders', _new_gotHeaders) + +def patch_HTTPPageGetter_handleResponse(): + def _handleResponse(self, response): + if self.quietLoss: + return + if self.failed: + self.factory.noPage( + failure.Failure( + error.Error( + self.status, self.message, response))) + if self.factory.method.upper() == 'HEAD': + # Callback with empty string, since there is never a response + # body for HEAD requests. + self.factory.page('') + elif self.length != None and self.length != 0: + self.factory.noPage(failure.Failure( + PartialDownloadError(self.status, self.message, response))) + else: + self.factory.page(response) + # server might be stupid and not close connection. admittedly + # the fact we do only one request per connection is also + # stupid... + self.transport.loseConnection() + setattr(HTTPPageGetter, 'handleResponse', _handleResponse)