From 7748ee6bba8eacb889ce68cd6ced273255b8b9a9 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 5 Feb 2016 14:52:03 +0300 Subject: [PATCH] mock date in s3 tests when using botocore --- scrapy/core/downloader/handlers/s3.py | 8 +-- tests/test_downloader_handlers.py | 93 ++++++++++++++++++--------- 2 files changed, 67 insertions(+), 34 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 0903b84ad..cb2bb46b1 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -68,10 +68,10 @@ class S3DownloadHandler(object): except Exception as ex: raise NotConfigured(str(ex)) else: - SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] - # TODO - anon - self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) + if not self.anon: + SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] + self._signer = SignerCls(botocore.credentials.Credentials( + aws_access_key_id, aws_secret_access_key)) self._download_http = httpdownloadhandler(settings).download_request diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 0f79a208d..6c4d2e0db 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,10 @@ import os import six +import contextlib +try: + from unittest import mock +except ImportError: + import mock from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory @@ -433,16 +438,16 @@ class HttpDownloadHandlerMock(object): class BaseS3TestCase(unittest.TestCase): - if six.PY3: - try: - import botocore - except ImportError: + try: + import botocore + except ImportError: + if six.PY2: + try: + import boto + except ImportError: + skip = 'missing botocore or boto library' + else: skip = 'missing botocore library' - else: - try: - import boto - except ImportError: - skip = 'missing boto library' class S3AnonTestCase(BaseS3TestCase): @@ -464,7 +469,7 @@ class S3AnonTestCase(BaseS3TestCase): httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/') -class S3TestCase(unittest.TestCase): +class S3TestCase(BaseS3TestCase): download_handler_cls = S3DownloadHandler # test use same example keys than amazon developer guide @@ -481,63 +486,89 @@ class S3TestCase(unittest.TestCase): self.download_request = s3reqh.download_request self.spider = Spider('foo') + @contextlib.contextmanager + def _mocked_date(self, date): + try: + import botocore.auth + except ImportError: + yield + else: + # We need to mock botocore.auth.formatdate, because otherwise + # botocore overrides Date header with current date and time + # and Authorization header is different each time + with mock.patch('botocore.auth.formatdate') as mock_formatdate: + mock_formatdate.return_value = date + yield + def test_request_signing1(self): # gets an object from the johnsmith bucket. - req = Request('s3://johnsmith/photos/puppy.jpg', - headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'}) - httpreq = self.download_request(req, self.spider) + date ='Tue, 27 Mar 2007 19:36:42 +0000' + req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') def test_request_signing2(self): # puts an object into the johnsmith bucket. + date = 'Tue, 27 Mar 2007 21:15:45 +0000' req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ 'Content-Type': 'image/jpeg', - 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000', + 'Date': date, 'Content-Length': '94328', }) - httpreq = self.download_request(req, self.spider) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') def test_request_signing3(self): # lists the content of the johnsmith bucket. + date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \ method='GET', headers={ 'User-Agent': 'Mozilla/5.0', - 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', + 'Date': date, }) - httpreq = self.download_request(req, self.spider) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. - req = Request('s3://johnsmith/?acl', \ - method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'}) - httpreq = self.download_request(req, self.spider) + date = 'Tue, 27 Mar 2007 19:44:46 +0000' + req = Request('s3://johnsmith/?acl', + method='GET', headers={'Date': date}) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): # deletes an object from the 'johnsmith' bucket using the # path-style and Date alternative. + date = 'Tue, 27 Mar 2007 21:20:27 +0000' req = Request('s3://johnsmith/photos/puppy.jpg', \ method='DELETE', headers={ - 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000', + 'Date': date, 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', }) - httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) + # botocore does not override Date with x-amz-date + self.assertIn(httpreq.headers['Authorization'], [ + b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=', + b'AWS 0PN5J17HBGZHT7JJ3X82:otYM2krxnuHhAofO4oqIV7wcfdU=']) def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. + date = 'Tue, 27 Mar 2007 21:06:08 +0000' req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \ method='PUT', headers={ 'User-Agent': 'curl/7.15.5', 'Host': 'static.johnsmith.net:8080', - 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000', + 'Date': date, 'x-amz-acl': 'public-read', 'content-type': 'application/x-download', 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', @@ -548,20 +579,22 @@ class S3TestCase(unittest.TestCase): 'Content-Encoding': 'gzip', 'Content-Length': '5913339', }) - httpreq = self.download_request(req, self.spider) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual(httpreq.headers['Authorization'], \ b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') def test_request_signing7(self): # ensure that spaces are quoted properly before signing + date = 'Tue, 27 Mar 2007 19:42:41 +0000' req = Request( ("s3://johnsmith/photos/my puppy.jpg" "?response-content-disposition=my puppy.jpg"), method='GET', - headers={ - 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', - }) - httpreq = self.download_request(req, self.spider) + headers={'Date': date}, + ) + with self._mocked_date(date): + httpreq = self.download_request(req, self.spider) self.assertEqual( httpreq.headers['Authorization'], b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')