mirror of https://github.com/scrapy/scrapy.git
mock date in s3 tests when using botocore
This commit is contained in:
Konstantin Lopuhin
parent
c3fec83e7e
commit
7748ee6bba
|
|
@ -68,10 +68,10 @@ class S3DownloadHandler(object):
|
|||
except Exception as ex:
|
||||
raise NotConfigured(str(ex))
|
||||
else:
|
||||
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
|
||||
# TODO - anon
|
||||
self._signer = SignerCls(botocore.credentials.Credentials(
|
||||
aws_access_key_id, aws_secret_access_key))
|
||||
if not self.anon:
|
||||
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
|
||||
self._signer = SignerCls(botocore.credentials.Credentials(
|
||||
aws_access_key_id, aws_secret_access_key))
|
||||
|
||||
self._download_http = httpdownloadhandler(settings).download_request
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
import os
|
||||
import six
|
||||
import contextlib
|
||||
try:
|
||||
from unittest import mock
|
||||
except ImportError:
|
||||
import mock
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
|
|
@ -433,16 +438,16 @@ class HttpDownloadHandlerMock(object):
|
|||
|
||||
|
||||
class BaseS3TestCase(unittest.TestCase):
|
||||
if six.PY3:
|
||||
try:
|
||||
import botocore
|
||||
except ImportError:
|
||||
try:
|
||||
import botocore
|
||||
except ImportError:
|
||||
if six.PY2:
|
||||
try:
|
||||
import boto
|
||||
except ImportError:
|
||||
skip = 'missing botocore or boto library'
|
||||
else:
|
||||
skip = 'missing botocore library'
|
||||
else:
|
||||
try:
|
||||
import boto
|
||||
except ImportError:
|
||||
skip = 'missing boto library'
|
||||
|
||||
|
||||
class S3AnonTestCase(BaseS3TestCase):
|
||||
|
|
@ -464,7 +469,7 @@ class S3AnonTestCase(BaseS3TestCase):
|
|||
httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
|
||||
|
||||
|
||||
class S3TestCase(unittest.TestCase):
|
||||
class S3TestCase(BaseS3TestCase):
|
||||
download_handler_cls = S3DownloadHandler
|
||||
|
||||
# test use same example keys than amazon developer guide
|
||||
|
|
@ -481,63 +486,89 @@ class S3TestCase(unittest.TestCase):
|
|||
self.download_request = s3reqh.download_request
|
||||
self.spider = Spider('foo')
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _mocked_date(self, date):
|
||||
try:
|
||||
import botocore.auth
|
||||
except ImportError:
|
||||
yield
|
||||
else:
|
||||
# We need to mock botocore.auth.formatdate, because otherwise
|
||||
# botocore overrides Date header with current date and time
|
||||
# and Authorization header is different each time
|
||||
with mock.patch('botocore.auth.formatdate') as mock_formatdate:
|
||||
mock_formatdate.return_value = date
|
||||
yield
|
||||
|
||||
def test_request_signing1(self):
|
||||
# gets an object from the johnsmith bucket.
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg',
|
||||
headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
date ='Tue, 27 Mar 2007 19:36:42 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
|
||||
|
||||
def test_request_signing2(self):
|
||||
# puts an object into the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 21:15:45 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
|
||||
'Content-Type': 'image/jpeg',
|
||||
'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
|
||||
'Date': date,
|
||||
'Content-Length': '94328',
|
||||
})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
|
||||
|
||||
def test_request_signing3(self):
|
||||
# lists the content of the johnsmith bucket.
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
|
||||
method='GET', headers={
|
||||
'User-Agent': 'Mozilla/5.0',
|
||||
'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
|
||||
'Date': date,
|
||||
})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
|
||||
|
||||
def test_request_signing4(self):
|
||||
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
|
||||
req = Request('s3://johnsmith/?acl', \
|
||||
method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
date = 'Tue, 27 Mar 2007 19:44:46 +0000'
|
||||
req = Request('s3://johnsmith/?acl',
|
||||
method='GET', headers={'Date': date})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
|
||||
def test_request_signing5(self):
|
||||
# deletes an object from the 'johnsmith' bucket using the
|
||||
# path-style and Date alternative.
|
||||
date = 'Tue, 27 Mar 2007 21:20:27 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', \
|
||||
method='DELETE', headers={
|
||||
'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
|
||||
'Date': date,
|
||||
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
|
||||
})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
# botocore does not override Date with x-amz-date
|
||||
self.assertIn(httpreq.headers['Authorization'], [
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=',
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU='])
|
||||
|
||||
def test_request_signing6(self):
|
||||
# uploads an object to a CNAME style virtual hosted bucket with metadata.
|
||||
date = 'Tue, 27 Mar 2007 21:06:08 +0000'
|
||||
req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
|
||||
method='PUT', headers={
|
||||
'User-Agent': 'curl/7.15.5',
|
||||
'Host': 'static.johnsmith.net:8080',
|
||||
'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
|
||||
'Date': date,
|
||||
'x-amz-acl': 'public-read',
|
||||
'content-type': 'application/x-download',
|
||||
'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
|
||||
|
|
@ -548,20 +579,22 @@ class S3TestCase(unittest.TestCase):
|
|||
'Content-Encoding': 'gzip',
|
||||
'Content-Length': '5913339',
|
||||
})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(httpreq.headers['Authorization'], \
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
|
||||
|
||||
def test_request_signing7(self):
|
||||
# ensure that spaces are quoted properly before signing
|
||||
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
|
||||
req = Request(
|
||||
("s3://johnsmith/photos/my puppy.jpg"
|
||||
"?response-content-disposition=my puppy.jpg"),
|
||||
method='GET',
|
||||
headers={
|
||||
'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
|
||||
})
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
headers={'Date': date},
|
||||
)
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
self.assertEqual(
|
||||
httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
|
||||
|
|
|
|||
Loading…
Reference in New Issue