MONKEYPATCH: in twisted < 8.0.0, HTTPPageGetter class can not handle HEAD requests because of response empty body raising PartialDownloadError.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40453

scrapy/trunk/scrapy/patches/monkeypatches.py

@@ -3,7 +3,8 @@ Monkey patches
 
 These are generally a bad idea.
 """
-from twisted.web.client import HTTPClientFactory
+from twisted.web.client import HTTPClientFactory, HTTPPageGetter
+import twisted
 
 
 # Extend limit for BeautifulSoup parsing loops
@@ -13,6 +14,9 @@ from twisted.web.client import HTTPClientFactory
 def apply_patches():
     patch_HTTPClientFactory_gotHeaders()
 
+    if twisted.__version__ < '8.0.0':
+        patch_HTTPPageGetter_handleResponse()
+
 
 # XXX: Monkeypatch for twisted.web.client-HTTPClientFactory
 # HTTPClientFactory.gotHeaders dies when parsing malformed cookies,
@@ -43,3 +47,27 @@ def _new_gotHeaders(self, headers):
 
 def patch_HTTPClientFactory_gotHeaders():
     setattr(HTTPClientFactory, 'gotHeaders', _new_gotHeaders)
+
+def patch_HTTPPageGetter_handleResponse():
+    def _handleResponse(self, response):
+        if self.quietLoss:
+            return
+        if self.failed:
+            self.factory.noPage(
+                failure.Failure(
+                    error.Error(
+                        self.status, self.message, response)))
+        if self.factory.method.upper() == 'HEAD':
+            # Callback with empty string, since there is never a response
+            # body for HEAD requests.
+            self.factory.page('')
+        elif self.length != None and self.length != 0:
+            self.factory.noPage(failure.Failure(
+                PartialDownloadError(self.status, self.message, response)))
+        else:
+            self.factory.page(response)
+        # server might be stupid and not close connection. admittedly
+        # the fact we do only one request per connection is also
+        # stupid...
+        self.transport.loseConnection()
+    setattr(HTTPPageGetter, 'handleResponse', _handleResponse)