moved HTTP auth functionality out of Request class and into scrapy.utils.request.request_authenticate function, added tests

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40726

scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py

@@ -1,8 +1,17 @@
+"""
+HTTP basic auth downloader middleware
+
+See documentation in docs/ref/downloader-middleware.rst
+"""
+
+from scrapy.utils.request import request_authenticate
+
 class HttpAuthMiddleware(object):
     """This middleware allows spiders to use HTTP auth in a cleaner way
     (http_user and http_pass spider class attributes)"""
 
     def process_request(self, request, spider):
-        if getattr(spider, 'http_user', None) or getattr(spider, 'http_pass', None):
-            request.httpauth(spider.http_user, spider.http_pass)
-
+        http_user = getattr(spider, 'http_user', '')
+        http_pass = getattr(spider, 'http_pass', '')
+        if http_user or http_pass:
+            request_authenticate(request, http_user, http_pass)

scrapy/trunk/scrapy/http/request.py

@@ -1,19 +1,18 @@
 import urllib
 from copy import copy
-from base64 import urlsafe_b64encode
 
 from twisted.internet import defer
 
 from scrapy.http.url import Url
 from scrapy.http.headers import Headers
 from scrapy.utils.url import safe_url_string
-from scrapy.utils.c14n import canonicalize
 from scrapy.utils.defer import chain_deferred
 
 class Request(object):
-    def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None,
-            referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None, 
-            domain=None):
+
+    def __init__(self, url, callback=None, context=None, method=None,
+        body=None, headers=None, cookies=None, referer=None,
+        url_encoding='utf-8', link_text='', dont_filter=None, domain=None):
 
         self.encoding = url_encoding  # this one has to be set first
         self.set_url(url)
@@ -22,7 +21,8 @@ class Request(object):
         if method is None and body is not None:
             method = 'POST' # backwards compatibility
         self.method = method.upper() if method else 'GET'
-        assert isinstance(self.method, basestring), 'Request method argument must be str or unicode, got %s: %s' % (type(method), method)
+        assert isinstance(self.method, basestring), \
+             'Request method argument must be str or unicode, got %s: %s' % (type(method), method)
 
         # body
         if isinstance(body, dict):
@@ -45,9 +45,6 @@ class Request(object):
         # shortcut for setting referer
         if referer is not None:
             self.headers['referer'] = referer
-        # http auth
-        if http_user or http_pass:
-            self.httpauth(http_user, http_pass)
         self.depth = 0
         self.link_text = link_text
         #allows to directly specify the spider for the request
@@ -74,13 +71,6 @@ class Request(object):
         self._url = Url(safe_url_string(decoded_url, self.encoding))
     url = property(lambda x: x._url, set_url)
 
-    def httpauth(self, http_user, http_pass):
-        if not http_user:
-            http_user = ''
-        if not http_pass:
-            http_pass = ''
-        self.headers['Authorization'] = 'Basic ' + urlsafe_b64encode("%s:%s" % (http_user, http_pass))
-
     def __str__(self):
         if self.method != 'GET':
             return "<(%s) %s>" % (self.method, self.url)

scrapy/trunk/scrapy/tests/test_utils_request.py

@@ -1,6 +1,6 @@
 import unittest
 from scrapy.http import Request
-from scrapy.utils.request import request_fingerprint
+from scrapy.utils.request import request_fingerprint, request_authenticate
 
 class UtilsRequestTest(unittest.TestCase):
 
@@ -55,7 +55,10 @@ class UtilsRequestTest(unittest.TestCase):
         fp2 = request_fingerprint(r2)
         self.assertNotEqual(fp1, fp2)
 
+    def test_request_authenticate(self):
+        r = Request("http://www.example.com")
+        request_authenticate(r, 'someuser', 'somepass')
+        self.assertEqual(r.headers['Authorization'], 'Basic c29tZXVzZXI6c29tZXBhc3M=')
+
 if __name__ == "__main__":
     unittest.main()
-
-

scrapy/trunk/scrapy/utils/request.py

@@ -4,6 +4,7 @@ scrapy.http.Request objects
 """
 
 import hashlib
+from base64 import urlsafe_b64encode
 
 from scrapy.utils.url import canonicalize_url
 
@@ -56,6 +57,13 @@ def request_fingerprint(request, include_headers=()):
         request._cache[cachekey] = fphash
         return fphash
 
+def request_authenticate(request, username, password):
+    """Autenticate the given request (in place) using the HTTP basic access
+    authentication mechanism (RFC 2617) and the given username and password
+    """
+    b64userpass = urlsafe_b64encode("%s:%s" % (username, password))
+    request.headers['Authorization'] = 'Basic ' + b64userpass
+
 def request_info(request):
     """Return a short string with request info including method, url and
     fingeprint. Mainly used for debugging