Fix assert_aws_environ: check for botocore with boto fallback on PY2

scrapy/utils/test.py

@@ -5,6 +5,7 @@ This module contains some assorted functions used in tests
 import os
 
 from importlib import import_module
+import six
 from twisted.trial.unittest import SkipTest
 
 
@@ -12,14 +13,22 @@ def assert_aws_environ():
     """Asserts the current environment is suitable for running AWS testsi.
     Raises SkipTest with the reason if it's not.
     """
-    try:
-        import boto
-    except ImportError as e:
-        raise SkipTest(str(e))
-
+    skip_if_no_boto()
     if 'AWS_ACCESS_KEY_ID' not in os.environ:
         raise SkipTest("AWS keys not found")
 
+def skip_if_no_boto():
+    try:
+        import botocore
+    except ImportError:
+        if six.PY2:
+            try:
+                import boto
+            except ImportError:
+                raise SkipTest('missing botocore or boto library')
+        else:
+            raise SkipTest('missing botocore library')
+
 def get_crawler(spidercls=None, settings_dict=None):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level

tests/test_downloader_handlers.py

@@ -28,7 +28,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.settings import Settings
-from scrapy.utils.test import get_crawler
+from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
@@ -437,22 +437,10 @@ class HttpDownloadHandlerMock(object):
         return request
 
 
-class BaseS3TestCase(unittest.TestCase):
-    try:
-        import botocore
-    except ImportError:
-        if six.PY2:
-            try:
-                import boto
-            except ImportError:
-                skip = 'missing botocore or boto library'
-        else:
-            skip = 'missing botocore library'
-
-
-class S3AnonTestCase(BaseS3TestCase):
+class S3AnonTestCase(unittest.TestCase):
 
     def setUp(self):
+        skip_if_no_boto()
         self.s3reqh = S3DownloadHandler(Settings(),
                 httpdownloadhandler=HttpDownloadHandlerMock,
                 #anon=True, # is implicit
@@ -469,7 +457,7 @@ class S3AnonTestCase(BaseS3TestCase):
             httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
-class S3TestCase(BaseS3TestCase):
+class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
 
     # test use same example keys than amazon developer guide
@@ -480,6 +468,7 @@ class S3TestCase(BaseS3TestCase):
     AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
 
     def setUp(self):
+        skip_if_no_boto()
         s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
                 self.AWS_SECRET_ACCESS_KEY,
                 httpdownloadhandler=HttpDownloadHandlerMock)