diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index d2f7c0ae4..43abd64a0 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,6 +2,7 @@ This module contains some assorted functions used in tests """ +from __future__ import absolute_import import os from importlib import import_module @@ -25,6 +26,25 @@ def skip_if_no_boto(): except NotConfigured as e: raise SkipTest(e.message) +def get_s3_content_and_delete(bucket, path): + """ Get content from s3 key, and delete key afterwards. + """ + if is_botocore(): + import botocore.session + session = botocore.session.get_session() + client = session.create_client('s3') + key = client.get_object(Bucket=bucket, Key=path) + content = key['Body'].read() + client.delete_object(Bucket=bucket, Key=path) + else: + import boto + # assuming boto=2.2.2 + bucket = boto.connect_s3().get_bucket(bucket, validate=False) + key = bucket.get_key(path) + content = key.get_contents_as_string() + bucket.delete_key(path) + return content + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index f3cf1c2cb..fd2f5a2ba 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage ) -from scrapy.utils.test import assert_aws_environ +from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete from scrapy.utils.python import to_native_str from scrapy.utils.boto import is_botocore @@ -93,7 +93,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): assert_aws_environ() - uri = os.environ.get('FEEDTEST_S3_URI') + uri = os.environ.get('S3_TEST_FILE_URI') if not uri: raise unittest.SkipTest("No S3 URI available for testing") storage = S3FeedStorage(uri) @@ -103,25 +103,9 @@ class S3FeedStorageTest(unittest.TestCase): file.write(expected_content) yield storage.store(file) u = urlparse(uri) - content = self._get_content_and_delete(u.hostname, u.path[1:]) + content = get_s3_content_and_delete(u.hostname, u.path[1:]) self.assertEqual(content, expected_content) - def _get_content_and_delete(self, bucket, path): - if is_botocore(): - import botocore.session - session = botocore.session.get_session() - client = session.create_client('s3') - key = client.get_object(Bucket=bucket, Key=path) - content = key['Body'].read() - client.delete_object(Bucket=bucket, Key=path) - else: - from boto import connect_s3 - bucket = connect_s3().get_bucket(bucket, validate=False) - key = bucket.get_key(path) - content = key.get_contents_as_string() - bucket.delete_key(path) - return content - class StdoutFeedStorageTest(unittest.TestCase): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index c9977f5ca..6ea47086f 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -4,15 +4,18 @@ import hashlib import warnings from tempfile import mkdtemp from shutil import rmtree +from six.moves.urllib.parse import urlparse +from six import BytesIO from twisted.trial import unittest from twisted.internet import defer -from scrapy.pipelines.files import FilesPipeline, FSFilesStore +from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore from scrapy.item import Item, Field from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.utils.python import to_bytes +from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete from tests import mock @@ -179,6 +182,28 @@ class FilesPipelineTestCaseFields(unittest.TestCase): self.assertEqual(item['stored_file'], [results[0][1]]) +class TestS3FilesStore(unittest.TestCase): + @defer.inlineCallbacks + def test_persist(self): + assert_aws_environ() + uri = os.environ.get('S3_TEST_FILE_URI') + if not uri: + raise unittest.SkipTest("No S3 URI available for testing") + data = b"TestS3FilesStore: \xe2\x98\x83" + buf = BytesIO(data) + meta = {'foo': 'bar'} + path = '' + store = S3FilesStore(uri) + yield store.persist_file(path, buf, info=None, meta=meta) + s = yield store.stat_file(path, info=None) + self.assertIn('last_modified', s) + self.assertIn('checksum', s) + self.assertEqual(s['checksum'], b'3187896a9657a28163abb31667df64c8') + u = urlparse(uri) + content = get_s3_content_and_delete(u.hostname, u.path[1:]) + self.assertEqual(content, data) + + class ItemWithFiles(Item): file_urls = Field() files = Field() diff --git a/tox.ini b/tox.ini index 4d8236b4e..2a8067618 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ deps = leveldb -rtests/requirements.txt passenv = - FEEDTEST_S3_URI + S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY commands =