add direct test for S3FilesStore

scrapy/utils/test.py

@@ -2,6 +2,7 @@
 This module contains some assorted functions used in tests
 """
 
+from __future__ import absolute_import
 import os
 
 from importlib import import_module
@@ -25,6 +26,25 @@ def skip_if_no_boto():
     except NotConfigured as e:
         raise SkipTest(e.message)
 
+def get_s3_content_and_delete(bucket, path):
+    """ Get content from s3 key, and delete key afterwards.
+    """
+    if is_botocore():
+        import botocore.session
+        session = botocore.session.get_session()
+        client = session.create_client('s3')
+        key = client.get_object(Bucket=bucket, Key=path)
+        content = key['Body'].read()
+        client.delete_object(Bucket=bucket, Key=path)
+    else:
+        import boto
+        # assuming boto=2.2.2
+        bucket = boto.connect_s3().get_bucket(bucket, validate=False)
+        key = bucket.get_key(path)
+        content = key.get_contents_as_string()
+        bucket.delete_key(path)
+    return content
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level

tests/test_feedexport.py

@@ -20,7 +20,7 @@ from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
     S3FeedStorage, StdoutFeedStorage
 )
-from scrapy.utils.test import assert_aws_environ
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 from scrapy.utils.python import to_native_str
 from scrapy.utils.boto import is_botocore
 
@@ -93,7 +93,7 @@ class S3FeedStorageTest(unittest.TestCase):
     @defer.inlineCallbacks
     def test_store(self):
         assert_aws_environ()
-        uri = os.environ.get('FEEDTEST_S3_URI')
+        uri = os.environ.get('S3_TEST_FILE_URI')
         if not uri:
             raise unittest.SkipTest("No S3 URI available for testing")
         storage = S3FeedStorage(uri)
@@ -103,25 +103,9 @@ class S3FeedStorageTest(unittest.TestCase):
         file.write(expected_content)
         yield storage.store(file)
         u = urlparse(uri)
-        content = self._get_content_and_delete(u.hostname, u.path[1:])
+        content = get_s3_content_and_delete(u.hostname, u.path[1:])
         self.assertEqual(content, expected_content)
 
-    def _get_content_and_delete(self, bucket, path):
-        if is_botocore():
-            import botocore.session
-            session = botocore.session.get_session()
-            client = session.create_client('s3')
-            key = client.get_object(Bucket=bucket, Key=path)
-            content = key['Body'].read()
-            client.delete_object(Bucket=bucket, Key=path)
-        else:
-            from boto import connect_s3
-            bucket = connect_s3().get_bucket(bucket, validate=False)
-            key = bucket.get_key(path)
-            content = key.get_contents_as_string()
-            bucket.delete_key(path)
-        return content
-
 
 class StdoutFeedStorageTest(unittest.TestCase):
 

tests/test_pipeline_files.py

@@ -4,15 +4,18 @@ import hashlib
 import warnings
 from tempfile import mkdtemp
 from shutil import rmtree
+from six.moves.urllib.parse import urlparse
+from six import BytesIO
 
 from twisted.trial import unittest
 from twisted.internet import defer
 
-from scrapy.pipelines.files import FilesPipeline, FSFilesStore
+from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
 
 from tests import mock
 
@@ -179,6 +182,28 @@ class FilesPipelineTestCaseFields(unittest.TestCase):
             self.assertEqual(item['stored_file'], [results[0][1]])
 
 
+class TestS3FilesStore(unittest.TestCase):
+    @defer.inlineCallbacks
+    def test_persist(self):
+        assert_aws_environ()
+        uri = os.environ.get('S3_TEST_FILE_URI')
+        if not uri:
+            raise unittest.SkipTest("No S3 URI available for testing")
+        data = b"TestS3FilesStore: \xe2\x98\x83"
+        buf = BytesIO(data)
+        meta = {'foo': 'bar'}
+        path = ''
+        store = S3FilesStore(uri)
+        yield store.persist_file(path, buf, info=None, meta=meta)
+        s = yield store.stat_file(path, info=None)
+        self.assertIn('last_modified', s)
+        self.assertIn('checksum', s)
+        self.assertEqual(s['checksum'], b'3187896a9657a28163abb31667df64c8')
+        u = urlparse(uri)
+        content = get_s3_content_and_delete(u.hostname, u.path[1:])
+        self.assertEqual(content, data)
+
+
 class ItemWithFiles(Item):
     file_urls = Field()
     files = Field()

tox.ini

@@ -15,7 +15,7 @@ deps =
     leveldb
     -rtests/requirements.txt
 passenv =
-    FEEDTEST_S3_URI
+    S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
 commands =