From 072fed8d4acc21c08bd28e7e1397f2f9d7a7c961 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 25 Nov 2008 11:06:34 +0000 Subject: [PATCH] aws: request signing function and tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40416 --- scrapy/trunk/scrapy/contrib/aws.py | 45 +++----- .../trunk/scrapy/contrib/pipeline/s3images.py | 15 +-- scrapy/trunk/scrapy/tests/test_aws.py | 109 ++++++++++++++++++ 3 files changed, 128 insertions(+), 41 deletions(-) create mode 100644 scrapy/trunk/scrapy/tests/test_aws.py diff --git a/scrapy/trunk/scrapy/contrib/aws.py b/scrapy/trunk/scrapy/contrib/aws.py index b821ea7d9..e81b9aa1f 100644 --- a/scrapy/trunk/scrapy/contrib/aws.py +++ b/scrapy/trunk/scrapy/contrib/aws.py @@ -3,6 +3,7 @@ import time import hmac import base64 import hashlib +from urlparse import urlsplit METADATA_PREFIX = 'x-amz-meta-' @@ -14,17 +15,15 @@ def canonical_string(method, path, headers, expires=None): interesting_headers = {} for key in headers: lk = key.lower() - if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX): + if lk in set('content-md5', 'content-type', 'date') or lk.startswith(AMAZON_HEADER_PREFIX): interesting_headers[lk] = headers[key].strip() # these keys get empty strings if they don't exist - if not interesting_headers.has_key('content-type'): - interesting_headers['content-type'] = '' - if not interesting_headers.has_key('content-md5'): - interesting_headers['content-md5'] = '' + interesting_headers.setdefault('content-type', '') + interesting_headers.setdefault('content-md5', '') # just in case someone used this. it's not necessary in this lib. - if interesting_headers.has_key('x-amz-date'): + if 'x-amz-date' in interesting_headers: interesting_headers['date'] = '' # if you're using expires for query string auth, then it trumps date @@ -58,32 +57,20 @@ def canonical_string(method, path, headers, expires=None): return buf -def merge_meta(headers, metadata): - final_headers = headers.copy() - for k in metadata.keys(): - if k.lower() in ['content-md5', 'content-type', 'date']: - final_headers[k] = metadata[k] - else: - final_headers[METADATA_PREFIX + k] = metadata[k] - return final_headers +def sign_request(req, accesskey, secretkey): + if 'Date' not in req.headers: + req.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) -def get_aws_metadata(headers): - metadata = {} - for hkey in headers.keys(): - if hkey.lower().startswith(METADATA_PREFIX): - metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey] - del headers[hkey] - return metadata + parsed = urlsplit(req.url) + bucket = parsed.hostname.replace('.s3.amazonaws.com','') + key = '%s?%s' % (parsed.path, parsed.query) if parsed.query else parsed.path + fqkey = '/%s%s' % (bucket, key) - -def add_aws_auth_header(headers, method, path): - if not headers.has_key('Date'): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - - c_string = canonical_string(method, path, headers) - _hmac = hmac.new(settings.AWS_SECRET_ACCESS_KEY, digestmod=hashlib.sha1) + c_string = canonical_string(req.method, fqkey, req.headers) + _hmac = hmac.new(secretkey, digestmod=hashlib.sha1) _hmac.update(c_string) b64_hmac = base64.encodestring(_hmac.digest()).strip() - headers['Authorization'] = "AWS %s:%s" % (settings.AWS_ACCESS_KEY_ID, b64_hmac) + req.headers['Authorization'] = "AWS %s:%s" % (accesskey, b64_hmac) + diff --git a/scrapy/trunk/scrapy/contrib/pipeline/s3images.py b/scrapy/trunk/scrapy/contrib/pipeline/s3images.py index 5b13471f3..584061e82 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/s3images.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/s3images.py @@ -12,7 +12,7 @@ from scrapy.http import Request from scrapy.stats import stats from scrapy.core.exceptions import DropItem, NotConfigured, HttpException from scrapy.contrib.pipeline.media import MediaPipeline -from scrapy.contrib.aws import canonical_string +from scrapy.contrib.aws import canonical_string, sign_request from scrapy.conf import settings from .images import BaseImagesPipeline, NoimagesDrop, ImageException @@ -33,23 +33,14 @@ class S3ImagesPipeline(BaseImagesPipeline): self.bucket_name = settings['S3_BUCKET'] self.prefix = settings['S3_PREFIX'] self.access_key = settings['AWS_ACCESS_KEY_ID'] + self.secret_key = settings['AWS_SECRET_ACCESS_KEY'] self.image_refresh_days = settings.getint('IMAGES_REFRESH_DAYS', 90) - self._hmac = hmac.new(settings['AWS_SECRET_ACCESS_KEY'], digestmod=hashlib.sha1) MediaPipeline.__init__(self) def s3request(self, key, method, body=None, headers=None): url = 'http://%s.s3.amazonaws.com/%s' % (self.bucket_name, key) req = Request(url, method=method, body=body, headers=headers) - - if not (headers and 'Date' in headers): - req.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) - - fullkey = '/%s/%s' % (self.bucket_name, key) - c_string = canonical_string(method, fullkey, req.headers) - _hmac = self._hmac.copy() - _hmac.update(c_string) - b64_hmac = base64.encodestring(_hmac.digest()).strip() - req.headers['Authorization'] = "AWS %s:%s" % (self.access_key, b64_hmac) + sign_request(req, self.access_key, self.secret_keself.secret_keyy) return req def image_downloaded(self, response, request, info): diff --git a/scrapy/trunk/scrapy/tests/test_aws.py b/scrapy/trunk/scrapy/tests/test_aws.py new file mode 100644 index 000000000..71fc45e5f --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_aws.py @@ -0,0 +1,109 @@ +from unittest import TestCase, main +from scrapy.contrib import aws +from scrapy.http import Request + +# keys are provided by amazon developer guide at +# http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf +# and the tests described here are the examples from that manual + +AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82' +AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' + + +class ScrapyAWSTest(TestCase): + def test_cannonical_string1(self): + cs = aws.canonical_string('GET', '/johnsmith/photos/puppy.jpg', { + 'Host': 'johnsmith.s3.amazonaws.com', + 'Date': 'Tue, 27 Mar 2007 19:36:42 +0000', + }) + self.assertEqual(cs, \ + '''GET\n\n\nTue, 27 Mar 2007 19:36:42 +0000\n/johnsmith/photos/puppy.jpg''') + + def test_cannonical_string2(self): + cs = aws.canonical_string('PUT', '/johnsmith/photos/puppy.jpg', { + 'Content-Type': 'image/jpeg', + 'Host': 'johnsmith.s3.amazonaws.com', + 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000', + 'Content-Length': '94328', + }) + self.assertEqual(cs, \ + '''PUT\n\nimage/jpeg\nTue, 27 Mar 2007 21:15:45 +0000\n/johnsmith/photos/puppy.jpg''') + + def test_request_signing1(self): + # gets an object from the johnsmith bucket. + req = Request('http://johnsmith.s3.amazonaws.com/photos/puppy.jpg', headers={ + 'Date': 'Tue, 27 Mar 2007 19:36:42 +0000', + }) + aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + self.assertEqual(req.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + + def test_request_signing2(self): + # puts an object into the johnsmith bucket. + req = Request('http://johnsmith.s3.amazonaws.com/photos/puppy.jpg', method='PUT', headers={ + 'Content-Type': 'image/jpeg', + 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000', + 'Content-Length': '94328', + }) + aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + self.assertEqual(req.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + + def test_request_signing3(self): + # lists the content of the johnsmith bucket. + req = Request('http://johnsmith.s3.amazonaws.com/?prefix=photos&max-keys=50&marker=puppy', \ + method='GET', headers={ + 'User-Agent': 'Mozilla/5.0', + 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', + }) + aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + self.assertEqual(req.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') + + def test_request_signing4(self): + # fetches the access control policy sub-resource for the 'johnsmith' bucket. + req = Request('http://johnsmith.s3.amazonaws.com/?acl', \ + method='GET', headers={ + 'Date': 'Tue, 27 Mar 2007 19:44:46 +0000', + }) + aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + self.assertEqual(req.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') + + def test_request_signing5(self): + # deletes an object from the 'johnsmith' bucket using the path-style and Date alternative. + req = Request('http://johnsmith.s3.amazonaws.com/photos/puppy.jpg', \ + method='DELETE', headers={ + 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000', + 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', + }) + aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + self.assertEqual(req.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + + def test_request_signing6(self): + # uploads an object to a CNAME style virtual hosted bucket with metadata. + req = Request('http://static.johnsmith.net:8080/db-backup.dat.gz', \ + method='PUT', headers={ + 'User-Agent': 'curl/7.15.5', + 'Host': 'static.johnsmith.net:8080', + 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000', + 'x-amz-acl': 'public-read', + 'content-type': 'application/x-download', + 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', + 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', + 'X-Amz-Meta-FileChecksum': '0x02661779', + 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', + 'Content-Disposition': 'attachment; filename=database.dat', + 'Content-Encoding': 'gzip', + 'Content-Length': '5913339', + }) + aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) + self.assertEqual(req.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') + + + + +if __name__ == '__main__': + main()