From 1272b138ea3dd028846ab209cdfcaed5c66ff63f Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 14 Jan 2009 22:02:58 +0000 Subject: [PATCH] moved HTTP auth functionality out of Request class and into scrapy.utils.request.request_authenticate function, added tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40726 --- .../contrib/downloadermiddleware/httpauth.py | 15 ++++++++++--- scrapy/trunk/scrapy/http/request.py | 22 +++++-------------- .../trunk/scrapy/tests/test_utils_request.py | 9 +++++--- scrapy/trunk/scrapy/utils/request.py | 8 +++++++ 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py index fcc9ebb8c..e9acc8b37 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/httpauth.py @@ -1,8 +1,17 @@ +""" +HTTP basic auth downloader middleware + +See documentation in docs/ref/downloader-middleware.rst +""" + +from scrapy.utils.request import request_authenticate + class HttpAuthMiddleware(object): """This middleware allows spiders to use HTTP auth in a cleaner way (http_user and http_pass spider class attributes)""" def process_request(self, request, spider): - if getattr(spider, 'http_user', None) or getattr(spider, 'http_pass', None): - request.httpauth(spider.http_user, spider.http_pass) - + http_user = getattr(spider, 'http_user', '') + http_pass = getattr(spider, 'http_pass', '') + if http_user or http_pass: + request_authenticate(request, http_user, http_pass) diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index 1716af270..0e3f8675a 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -1,19 +1,18 @@ import urllib from copy import copy -from base64 import urlsafe_b64encode from twisted.internet import defer from scrapy.http.url import Url from scrapy.http.headers import Headers from scrapy.utils.url import safe_url_string -from scrapy.utils.c14n import canonicalize from scrapy.utils.defer import chain_deferred class Request(object): - def __init__(self, url, callback=None, context=None, method=None, body=None, headers=None, cookies=None, - referer=None, url_encoding='utf-8', link_text='', http_user='', http_pass='', dont_filter=None, - domain=None): + + def __init__(self, url, callback=None, context=None, method=None, + body=None, headers=None, cookies=None, referer=None, + url_encoding='utf-8', link_text='', dont_filter=None, domain=None): self.encoding = url_encoding # this one has to be set first self.set_url(url) @@ -22,7 +21,8 @@ class Request(object): if method is None and body is not None: method = 'POST' # backwards compatibility self.method = method.upper() if method else 'GET' - assert isinstance(self.method, basestring), 'Request method argument must be str or unicode, got %s: %s' % (type(method), method) + assert isinstance(self.method, basestring), \ + 'Request method argument must be str or unicode, got %s: %s' % (type(method), method) # body if isinstance(body, dict): @@ -45,9 +45,6 @@ class Request(object): # shortcut for setting referer if referer is not None: self.headers['referer'] = referer - # http auth - if http_user or http_pass: - self.httpauth(http_user, http_pass) self.depth = 0 self.link_text = link_text #allows to directly specify the spider for the request @@ -74,13 +71,6 @@ class Request(object): self._url = Url(safe_url_string(decoded_url, self.encoding)) url = property(lambda x: x._url, set_url) - def httpauth(self, http_user, http_pass): - if not http_user: - http_user = '' - if not http_pass: - http_pass = '' - self.headers['Authorization'] = 'Basic ' + urlsafe_b64encode("%s:%s" % (http_user, http_pass)) - def __str__(self): if self.method != 'GET': return "<(%s) %s>" % (self.method, self.url) diff --git a/scrapy/trunk/scrapy/tests/test_utils_request.py b/scrapy/trunk/scrapy/tests/test_utils_request.py index 60b4bdac5..6ecc07d51 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_request.py +++ b/scrapy/trunk/scrapy/tests/test_utils_request.py @@ -1,6 +1,6 @@ import unittest from scrapy.http import Request -from scrapy.utils.request import request_fingerprint +from scrapy.utils.request import request_fingerprint, request_authenticate class UtilsRequestTest(unittest.TestCase): @@ -55,7 +55,10 @@ class UtilsRequestTest(unittest.TestCase): fp2 = request_fingerprint(r2) self.assertNotEqual(fp1, fp2) + def test_request_authenticate(self): + r = Request("http://www.example.com") + request_authenticate(r, 'someuser', 'somepass') + self.assertEqual(r.headers['Authorization'], 'Basic c29tZXVzZXI6c29tZXBhc3M=') + if __name__ == "__main__": unittest.main() - - diff --git a/scrapy/trunk/scrapy/utils/request.py b/scrapy/trunk/scrapy/utils/request.py index 41d49c282..e9da7c9cf 100644 --- a/scrapy/trunk/scrapy/utils/request.py +++ b/scrapy/trunk/scrapy/utils/request.py @@ -4,6 +4,7 @@ scrapy.http.Request objects """ import hashlib +from base64 import urlsafe_b64encode from scrapy.utils.url import canonicalize_url @@ -56,6 +57,13 @@ def request_fingerprint(request, include_headers=()): request._cache[cachekey] = fphash return fphash +def request_authenticate(request, username, password): + """Autenticate the given request (in place) using the HTTP basic access + authentication mechanism (RFC 2617) and the given username and password + """ + b64userpass = urlsafe_b64encode("%s:%s" % (username, password)) + request.headers['Authorization'] = 'Basic ' + b64userpass + def request_info(request): """Return a short string with request info including method, url and fingeprint. Mainly used for debugging