mirror of https://github.com/scrapy/scrapy.git
aws: request signing function and tests
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40416
This commit is contained in:
Daniel Grana
parent
0fbb7579f6
commit
072fed8d4a
|
|
@ -3,6 +3,7 @@ import time
|
|||
import hmac
|
||||
import base64
|
||||
import hashlib
|
||||
from urlparse import urlsplit
|
||||
|
||||
|
||||
METADATA_PREFIX = 'x-amz-meta-'
|
||||
|
|
@ -14,17 +15,15 @@ def canonical_string(method, path, headers, expires=None):
|
|||
interesting_headers = {}
|
||||
for key in headers:
|
||||
lk = key.lower()
|
||||
if lk in ['content-md5', 'content-type', 'date'] or lk.startswith(AMAZON_HEADER_PREFIX):
|
||||
if lk in set('content-md5', 'content-type', 'date') or lk.startswith(AMAZON_HEADER_PREFIX):
|
||||
interesting_headers[lk] = headers[key].strip()
|
||||
|
||||
# these keys get empty strings if they don't exist
|
||||
if not interesting_headers.has_key('content-type'):
|
||||
interesting_headers['content-type'] = ''
|
||||
if not interesting_headers.has_key('content-md5'):
|
||||
interesting_headers['content-md5'] = ''
|
||||
interesting_headers.setdefault('content-type', '')
|
||||
interesting_headers.setdefault('content-md5', '')
|
||||
|
||||
# just in case someone used this. it's not necessary in this lib.
|
||||
if interesting_headers.has_key('x-amz-date'):
|
||||
if 'x-amz-date' in interesting_headers:
|
||||
interesting_headers['date'] = ''
|
||||
|
||||
# if you're using expires for query string auth, then it trumps date
|
||||
|
|
@ -58,32 +57,20 @@ def canonical_string(method, path, headers, expires=None):
|
|||
return buf
|
||||
|
||||
|
||||
def merge_meta(headers, metadata):
|
||||
final_headers = headers.copy()
|
||||
for k in metadata.keys():
|
||||
if k.lower() in ['content-md5', 'content-type', 'date']:
|
||||
final_headers[k] = metadata[k]
|
||||
else:
|
||||
final_headers[METADATA_PREFIX + k] = metadata[k]
|
||||
|
||||
return final_headers
|
||||
def sign_request(req, accesskey, secretkey):
|
||||
if 'Date' not in req.headers:
|
||||
req.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
|
||||
def get_aws_metadata(headers):
|
||||
metadata = {}
|
||||
for hkey in headers.keys():
|
||||
if hkey.lower().startswith(METADATA_PREFIX):
|
||||
metadata[hkey[len(METADATA_PREFIX):]] = headers[hkey]
|
||||
del headers[hkey]
|
||||
return metadata
|
||||
parsed = urlsplit(req.url)
|
||||
bucket = parsed.hostname.replace('.s3.amazonaws.com','')
|
||||
key = '%s?%s' % (parsed.path, parsed.query) if parsed.query else parsed.path
|
||||
fqkey = '/%s%s' % (bucket, key)
|
||||
|
||||
|
||||
def add_aws_auth_header(headers, method, path):
|
||||
if not headers.has_key('Date'):
|
||||
headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
|
||||
c_string = canonical_string(method, path, headers)
|
||||
_hmac = hmac.new(settings.AWS_SECRET_ACCESS_KEY, digestmod=hashlib.sha1)
|
||||
c_string = canonical_string(req.method, fqkey, req.headers)
|
||||
_hmac = hmac.new(secretkey, digestmod=hashlib.sha1)
|
||||
_hmac.update(c_string)
|
||||
b64_hmac = base64.encodestring(_hmac.digest()).strip()
|
||||
headers['Authorization'] = "AWS %s:%s" % (settings.AWS_ACCESS_KEY_ID, b64_hmac)
|
||||
req.headers['Authorization'] = "AWS %s:%s" % (accesskey, b64_hmac)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from scrapy.http import Request
|
|||
from scrapy.stats import stats
|
||||
from scrapy.core.exceptions import DropItem, NotConfigured, HttpException
|
||||
from scrapy.contrib.pipeline.media import MediaPipeline
|
||||
from scrapy.contrib.aws import canonical_string
|
||||
from scrapy.contrib.aws import canonical_string, sign_request
|
||||
from scrapy.conf import settings
|
||||
|
||||
from .images import BaseImagesPipeline, NoimagesDrop, ImageException
|
||||
|
|
@ -33,23 +33,14 @@ class S3ImagesPipeline(BaseImagesPipeline):
|
|||
self.bucket_name = settings['S3_BUCKET']
|
||||
self.prefix = settings['S3_PREFIX']
|
||||
self.access_key = settings['AWS_ACCESS_KEY_ID']
|
||||
self.secret_key = settings['AWS_SECRET_ACCESS_KEY']
|
||||
self.image_refresh_days = settings.getint('IMAGES_REFRESH_DAYS', 90)
|
||||
self._hmac = hmac.new(settings['AWS_SECRET_ACCESS_KEY'], digestmod=hashlib.sha1)
|
||||
MediaPipeline.__init__(self)
|
||||
|
||||
def s3request(self, key, method, body=None, headers=None):
|
||||
url = 'http://%s.s3.amazonaws.com/%s' % (self.bucket_name, key)
|
||||
req = Request(url, method=method, body=body, headers=headers)
|
||||
|
||||
if not (headers and 'Date' in headers):
|
||||
req.headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
|
||||
|
||||
fullkey = '/%s/%s' % (self.bucket_name, key)
|
||||
c_string = canonical_string(method, fullkey, req.headers)
|
||||
_hmac = self._hmac.copy()
|
||||
_hmac.update(c_string)
|
||||
b64_hmac = base64.encodestring(_hmac.digest()).strip()
|
||||
req.headers['Authorization'] = "AWS %s:%s" % (self.access_key, b64_hmac)
|
||||
sign_request(req, self.access_key, self.secret_keself.secret_keyy)
|
||||
return req
|
||||
|
||||
def image_downloaded(self, response, request, info):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,109 @@
|
|||
from unittest import TestCase, main
|
||||
from scrapy.contrib import aws
|
||||
from scrapy.http import Request
|
||||
|
||||
# keys are provided by amazon developer guide at
|
||||
# http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
|
||||
# and the tests described here are the examples from that manual
|
||||
|
||||
AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82'
|
||||
AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
|
||||
|
||||
|
||||
class ScrapyAWSTest(TestCase):
|
||||
def test_cannonical_string1(self):
|
||||
cs = aws.canonical_string('GET', '/johnsmith/photos/puppy.jpg', {
|
||||
'Host': 'johnsmith.s3.amazonaws.com',
|
||||
'Date': 'Tue, 27 Mar 2007 19:36:42 +0000',
|
||||
})
|
||||
self.assertEqual(cs, \
|
||||
'''GET\n\n\nTue, 27 Mar 2007 19:36:42 +0000\n/johnsmith/photos/puppy.jpg''')
|
||||
|
||||
def test_cannonical_string2(self):
|
||||
cs = aws.canonical_string('PUT', '/johnsmith/photos/puppy.jpg', {
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Host': 'johnsmith.s3.amazonaws.com',
|
||||
'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
self.assertEqual(cs, \
|
||||
'''PUT\n\nimage/jpeg\nTue, 27 Mar 2007 21:15:45 +0000\n/johnsmith/photos/puppy.jpg''')
|
||||
|
||||
def test_request_signing1(self):
|
||||
# gets an object from the johnsmith bucket.
|
||||
req = Request('http://johnsmith.s3.amazonaws.com/photos/puppy.jpg', headers={
|
||||
'Date': 'Tue, 27 Mar 2007 19:36:42 +0000',
|
||||
})
|
||||
aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
self.assertEqual(req.headers['Authorization'], \
|
||||
'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
|
||||
|
||||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
req = Request('http://johnsmith.s3.amazonaws.com/photos/puppy.jpg', method='PUT', headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
self.assertEqual(req.headers['Authorization'], \
|
||||
'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
|
||||
|
||||
def test_request_signing3(self):
|
||||
# lists the content of the johnsmith bucket.
|
||||
req = Request('http://johnsmith.s3.amazonaws.com/?prefix=photos&max-keys=50&marker=puppy', \
|
||||
method='GET', headers={
|
||||
'User-Agent': 'Mozilla/5.0',
|
||||
'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
|
||||
})
|
||||
aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
self.assertEqual(req.headers['Authorization'], \
|
||||
'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
|
||||
|
||||
def test_request_signing4(self):
|
||||
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
|
||||
req = Request('http://johnsmith.s3.amazonaws.com/?acl', \
|
||||
method='GET', headers={
|
||||
'Date': 'Tue, 27 Mar 2007 19:44:46 +0000',
|
||||
})
|
||||
aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
self.assertEqual(req.headers['Authorization'], \
|
||||
'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
|
||||
def test_request_signing5(self):
|
||||
# deletes an object from the 'johnsmith' bucket using the path-style and Date alternative.
|
||||
req = Request('http://johnsmith.s3.amazonaws.com/photos/puppy.jpg', \
|
||||
method='DELETE', headers={
|
||||
'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
|
||||
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
|
||||
})
|
||||
aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
self.assertEqual(req.headers['Authorization'], \
|
||||
'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
|
||||
|
||||
def test_request_signing6(self):
|
||||
# uploads an object to a CNAME style virtual hosted bucket with metadata.
|
||||
req = Request('http://static.johnsmith.net:8080/db-backup.dat.gz', \
|
||||
method='PUT', headers={
|
||||
'User-Agent': 'curl/7.15.5',
|
||||
'Host': 'static.johnsmith.net:8080',
|
||||
'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
|
||||
'x-amz-acl': 'public-read',
|
||||
'content-type': 'application/x-download',
|
||||
'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
|
||||
'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
|
||||
'X-Amz-Meta-FileChecksum': '0x02661779',
|
||||
'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
|
||||
'Content-Disposition': 'attachment; filename=database.dat',
|
||||
'Content-Encoding': 'gzip',
|
||||
'Content-Length': '5913339',
|
||||
})
|
||||
aws.sign_request(req, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
|
||||
self.assertEqual(req.headers['Authorization'], \
|
||||
'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue