mirror of https://github.com/scrapy/scrapy.git
Merge pull request #4804 from Gallaecio/mock-s3
Use mocking for S3 tests that currently need server credentials
This commit is contained in:
Andrey Rahmatullin
GitHub
commit
1aeda66435
|
|
@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest
|
|||
from scrapy.utils.boto import is_botocore_available
|
||||
|
||||
|
||||
def assert_aws_environ():
|
||||
"""Asserts the current environment is suitable for running AWS testsi.
|
||||
Raises SkipTest with the reason if it's not.
|
||||
"""
|
||||
skip_if_no_boto()
|
||||
if 'AWS_ACCESS_KEY_ID' not in os.environ:
|
||||
raise SkipTest("AWS keys not found")
|
||||
|
||||
|
||||
def assert_gcs_environ():
|
||||
if 'GCS_PROJECT_ID' not in os.environ:
|
||||
raise SkipTest("GCS_PROJECT_ID not found")
|
||||
|
|
@ -32,18 +23,6 @@ def skip_if_no_boto():
|
|||
raise SkipTest('missing botocore library')
|
||||
|
||||
|
||||
def get_s3_content_and_delete(bucket, path, with_key=False):
|
||||
""" Get content from s3 key, and delete key afterwards.
|
||||
"""
|
||||
import botocore.session
|
||||
session = botocore.session.get_session()
|
||||
client = session.create_client('s3')
|
||||
key = client.get_object(Bucket=bucket, Key=path)
|
||||
content = key['Body'].read()
|
||||
client.delete_object(Bucket=bucket, Key=path)
|
||||
return (content, key) if with_key else content
|
||||
|
||||
|
||||
def get_gcs_content_and_delete(bucket, path):
|
||||
from google.cloud import storage
|
||||
client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
|
||||
|
|
|
|||
|
|
@ -868,29 +868,6 @@ class S3TestCase(unittest.TestCase):
|
|||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
|
||||
def test_request_signing5(self):
|
||||
try:
|
||||
import botocore # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
raise unittest.SkipTest(
|
||||
'botocore does not support overriding date with x-amz-date')
|
||||
# deletes an object from the 'johnsmith' bucket using the
|
||||
# path-style and Date alternative.
|
||||
date = 'Tue, 27 Mar 2007 21:20:27 +0000'
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
|
||||
'Date': date,
|
||||
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
|
||||
})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
# botocore does not override Date with x-amz-date
|
||||
self.assertEqual(
|
||||
httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
|
||||
|
||||
def test_request_signing6(self):
|
||||
# uploads an object to a CNAME style virtual hosted bucket with metadata.
|
||||
date = 'Tue, 27 Mar 2007 21:06:08 +0000'
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from logging import getLogger
|
|||
from pathlib import Path
|
||||
from string import ascii_letters, digits
|
||||
from unittest import mock
|
||||
from urllib.parse import urljoin, urlparse, quote
|
||||
from urllib.parse import urljoin, quote
|
||||
from urllib.request import pathname2url
|
||||
|
||||
import lxml.etree
|
||||
|
|
@ -41,8 +41,6 @@ from scrapy.extensions.feedexport import (
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.test import (
|
||||
assert_aws_environ,
|
||||
get_s3_content_and_delete,
|
||||
get_crawler,
|
||||
mock_google_cloud_storage,
|
||||
skip_if_no_boto,
|
||||
|
|
@ -254,21 +252,42 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_store(self):
|
||||
assert_aws_environ()
|
||||
uri = os.environ.get('S3_TEST_FILE_URI')
|
||||
if not uri:
|
||||
raise unittest.SkipTest("No S3 URI available for testing")
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
storage = S3FeedStorage(uri, access_key, secret_key)
|
||||
skip_if_no_boto()
|
||||
|
||||
settings = {
|
||||
'AWS_ACCESS_KEY_ID': 'access_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'secret_key',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
bucket = 'mybucket'
|
||||
key = 'export.csv'
|
||||
storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}')
|
||||
verifyObject(IFeedStorage, storage)
|
||||
file = storage.open(scrapy.Spider("default"))
|
||||
expected_content = b"content: \xe2\x98\x83"
|
||||
file.write(expected_content)
|
||||
yield storage.store(file)
|
||||
u = urlparse(uri)
|
||||
content = get_s3_content_and_delete(u.hostname, u.path[1:])
|
||||
self.assertEqual(content, expected_content)
|
||||
|
||||
file = mock.MagicMock()
|
||||
from botocore.stub import Stubber
|
||||
with Stubber(storage.s3_client) as stub:
|
||||
stub.add_response(
|
||||
'put_object',
|
||||
expected_params={
|
||||
'Body': file,
|
||||
'Bucket': bucket,
|
||||
'Key': key,
|
||||
},
|
||||
service_response={},
|
||||
)
|
||||
|
||||
yield storage.store(file)
|
||||
|
||||
stub.assert_no_pending_responses()
|
||||
self.assertEqual(
|
||||
file.method_calls,
|
||||
[
|
||||
mock.call.seek(0),
|
||||
# The call to read does not happen with Stubber
|
||||
mock.call.close(),
|
||||
]
|
||||
)
|
||||
|
||||
def test_init_without_acl(self):
|
||||
storage = S3FeedStorage(
|
||||
|
|
@ -1533,46 +1552,53 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_s3_export(self):
|
||||
"""
|
||||
Test export of items into s3 bucket.
|
||||
S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
|
||||
to perform this test:
|
||||
[testenv]
|
||||
setenv =
|
||||
AWS_SECRET_ACCESS_KEY = ABCD
|
||||
AWS_ACCESS_KEY_ID = EFGH
|
||||
S3_TEST_BUCKET_NAME = IJKL
|
||||
"""
|
||||
try:
|
||||
import boto3
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("S3FeedStorage requires boto3")
|
||||
skip_if_no_boto()
|
||||
|
||||
assert_aws_environ()
|
||||
s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
if not s3_test_bucket_name:
|
||||
raise unittest.SkipTest("No S3 BUCKET available for testing")
|
||||
|
||||
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
|
||||
filename = ''.join(chars)
|
||||
prefix = f'tmp/{filename}'
|
||||
s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json'
|
||||
storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
|
||||
settings = Settings({
|
||||
'FEEDS': {
|
||||
s3_test_file_uri: {
|
||||
'format': 'json',
|
||||
},
|
||||
},
|
||||
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
|
||||
})
|
||||
bucket = 'mybucket'
|
||||
items = [
|
||||
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
|
||||
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
|
||||
self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
|
||||
]
|
||||
|
||||
class CustomS3FeedStorage(S3FeedStorage):
|
||||
|
||||
stubs = []
|
||||
|
||||
def open(self, *args, **kwargs):
|
||||
from botocore.stub import ANY, Stubber
|
||||
stub = Stubber(self.s3_client)
|
||||
stub.activate()
|
||||
CustomS3FeedStorage.stubs.append(stub)
|
||||
stub.add_response(
|
||||
'put_object',
|
||||
expected_params={
|
||||
'Body': ANY,
|
||||
'Bucket': bucket,
|
||||
'Key': ANY,
|
||||
},
|
||||
service_response={},
|
||||
)
|
||||
return super().open(*args, **kwargs)
|
||||
|
||||
key = 'export.csv'
|
||||
uri = f's3://{bucket}/{key}/%(batch_time)s.json'
|
||||
batch_item_count = 1
|
||||
settings = {
|
||||
'AWS_ACCESS_KEY_ID': 'access_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'secret_key',
|
||||
'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count,
|
||||
'FEED_STORAGES': {
|
||||
's3': CustomS3FeedStorage,
|
||||
},
|
||||
'FEEDS': {
|
||||
uri: {
|
||||
'format': 'json',
|
||||
},
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
storage = S3FeedStorage.from_crawler(crawler, uri)
|
||||
verifyObject(IFeedStorage, storage)
|
||||
|
||||
class TestSpider(scrapy.Spider):
|
||||
|
|
@ -1582,22 +1608,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
for item in items:
|
||||
yield item
|
||||
|
||||
s3 = boto3.resource('s3')
|
||||
my_bucket = s3.Bucket(s3_test_bucket_name)
|
||||
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
|
||||
|
||||
with MockServer() as s:
|
||||
with MockServer() as server:
|
||||
runner = CrawlerRunner(Settings(settings))
|
||||
TestSpider.start_urls = [s.url('/')]
|
||||
TestSpider.start_urls = [server.url('/')]
|
||||
yield runner.crawl(TestSpider)
|
||||
|
||||
for file_uri in my_bucket.objects.filter(Prefix=prefix):
|
||||
content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
|
||||
if not content and not items:
|
||||
break
|
||||
content = json.loads(content.decode('utf-8'))
|
||||
expected_batch, items = items[:batch_size], items[batch_size:]
|
||||
self.assertEqual(expected_batch, content)
|
||||
self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
|
||||
for stub in CustomS3FeedStorage.stubs[:-1]:
|
||||
stub.assert_no_pending_responses()
|
||||
|
||||
|
||||
class FeedExportInitTest(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
|
|
@ -23,11 +24,10 @@ from scrapy.pipelines.files import (
|
|||
)
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.test import (
|
||||
assert_aws_environ,
|
||||
assert_gcs_environ,
|
||||
get_ftp_content_and_delete,
|
||||
get_gcs_content_and_delete,
|
||||
get_s3_content_and_delete,
|
||||
skip_if_no_boto,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -414,32 +414,88 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
|
||||
|
||||
class TestS3FilesStore(unittest.TestCase):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_persist(self):
|
||||
assert_aws_environ()
|
||||
uri = os.environ.get('S3_TEST_FILE_URI')
|
||||
if not uri:
|
||||
raise unittest.SkipTest("No S3 URI available for testing")
|
||||
data = b"TestS3FilesStore: \xe2\x98\x83"
|
||||
buf = BytesIO(data)
|
||||
skip_if_no_boto()
|
||||
|
||||
bucket = 'mybucket'
|
||||
key = 'export.csv'
|
||||
uri = f's3://{bucket}/{key}'
|
||||
buffer = mock.MagicMock()
|
||||
meta = {'foo': 'bar'}
|
||||
path = ''
|
||||
content_type = 'image/png'
|
||||
|
||||
store = S3FilesStore(uri)
|
||||
yield store.persist_file(
|
||||
path, buf, info=None, meta=meta,
|
||||
headers={'Content-Type': 'image/png'})
|
||||
s = yield store.stat_file(path, info=None)
|
||||
self.assertIn('last_modified', s)
|
||||
self.assertIn('checksum', s)
|
||||
self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
|
||||
u = urlparse(uri)
|
||||
content, key = get_s3_content_and_delete(
|
||||
u.hostname, u.path[1:], with_key=True)
|
||||
self.assertEqual(content, data)
|
||||
self.assertEqual(key['Metadata'], {'foo': 'bar'})
|
||||
self.assertEqual(
|
||||
key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
|
||||
self.assertEqual(key['ContentType'], 'image/png')
|
||||
from botocore.stub import Stubber
|
||||
with Stubber(store.s3_client) as stub:
|
||||
stub.add_response(
|
||||
'put_object',
|
||||
expected_params={
|
||||
'ACL': S3FilesStore.POLICY,
|
||||
'Body': buffer,
|
||||
'Bucket': bucket,
|
||||
'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
|
||||
'ContentType': content_type,
|
||||
'Key': key,
|
||||
'Metadata': meta,
|
||||
},
|
||||
service_response={},
|
||||
)
|
||||
|
||||
yield store.persist_file(
|
||||
path,
|
||||
buffer,
|
||||
info=None,
|
||||
meta=meta,
|
||||
headers={'Content-Type': content_type},
|
||||
)
|
||||
|
||||
stub.assert_no_pending_responses()
|
||||
self.assertEqual(
|
||||
buffer.method_calls,
|
||||
[
|
||||
mock.call.seek(0),
|
||||
# The call to read does not happen with Stubber
|
||||
]
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_stat(self):
|
||||
skip_if_no_boto()
|
||||
|
||||
bucket = 'mybucket'
|
||||
key = 'export.csv'
|
||||
uri = f's3://{bucket}/{key}'
|
||||
checksum = '3187896a9657a28163abb31667df64c8'
|
||||
last_modified = datetime(2019, 12, 1)
|
||||
|
||||
store = S3FilesStore(uri)
|
||||
from botocore.stub import Stubber
|
||||
with Stubber(store.s3_client) as stub:
|
||||
stub.add_response(
|
||||
'head_object',
|
||||
expected_params={
|
||||
'Bucket': bucket,
|
||||
'Key': key,
|
||||
},
|
||||
service_response={
|
||||
'ETag': f'"{checksum}"',
|
||||
'LastModified': last_modified,
|
||||
},
|
||||
)
|
||||
|
||||
file_stats = yield store.stat_file('', info=None)
|
||||
self.assertEqual(
|
||||
file_stats,
|
||||
{
|
||||
'checksum': checksum,
|
||||
'last_modified': last_modified.timestamp(),
|
||||
},
|
||||
)
|
||||
|
||||
stub.assert_no_pending_responses()
|
||||
|
||||
|
||||
class TestGCSFilesStore(unittest.TestCase):
|
||||
|
|
|
|||
Loading…
Reference in New Issue