RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

S3FeedStorageTest: add botocore support, and organize boto/botocore checks

This commit is contained in:
Konstantin Lopuhin 2016-02-15 17:27:34 +03:00
parent 5d2f067458
commit 3ada45a9bb
5 changed files with 77 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
scrapy/core/downloader/handlers/s3.py
View File

@ -1,17 +1,13 @@
import six
from six.moves.urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
from scrapy.utils.boto import is_botocore
from .http import HTTPDownloadHandler
def get_s3_connection():
try:
from boto.s3.connection import S3Connection
except ImportError:
return None
def _get_boto_connection():
from boto.s3.connection import S3Connection
class _v19_S3Connection(S3Connection):
"""A dummy S3Connection wrapper that doesn't do any synchronous download"""
@ -53,21 +49,9 @@ class S3DownloadHandler(object):
self.anon = kw.get('anon')
self._signer = None
try:
if is_botocore():
import botocore.auth
import botocore.credentials
except ImportError:
if six.PY3:
raise NotConfigured("missing botocore library")
_S3Connection = get_s3_connection()
if _S3Connection is None:
raise NotConfigured("missing botocore or boto library")
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
else:
kw.pop('anon', None)
if kw:
raise TypeError('Unexpected keyword arguments: %s' % kw)
@ -75,6 +59,13 @@ class S3DownloadHandler(object):
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
else:
_S3Connection = _get_boto_connection()
try:
self.conn = _S3Connection(
aws_access_key_id, aws_secret_access_key, **kw)
except Exception as ex:
raise NotConfigured(str(ex))
self._download_http = httpdownloadhandler(settings).download_request

32
scrapy/extensions/feedexport.py
View File

@ -24,6 +24,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import load_object
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.python import without_none_values
from scrapy.utils.boto import is_botocore
logger = logging.getLogger(__name__)
@ -90,24 +91,33 @@ class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri):
from scrapy.conf import settings
try:
import boto
except ImportError:
raise NotConfigured
self.connect_s3 = boto.connect_s3
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or settings['AWS_ACCESS_KEY_ID']
self.secret_key = u.password or settings['AWS_SECRET_ACCESS_KEY']
self.keyname = u.path
self.is_botocore = is_botocore()
self.keyname = u.path[1:] # remove first "/"
if self.is_botocore:
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key)
else:
import boto
self.connect_s3 = boto.connect_s3
def _store_in_thread(self, file):
file.seek(0)
conn = self.connect_s3(self.access_key, self.secret_key)
bucket = conn.get_bucket(self.bucketname, validate=False)
key = bucket.new_key(self.keyname)
key.set_contents_from_file(file)
key.close()
if self.is_botocore:
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file)
else:
conn = self.connect_s3(self.access_key, self.secret_key)
bucket = conn.get_bucket(self.bucketname, validate=False)
key = bucket.new_key(self.keyname)
key.set_contents_from_file(file)
key.close()
class FTPFeedStorage(BlockingFeedStorage):

20
scrapy/utils/boto.py Normal file
View File

@ -0,0 +1,20 @@
"""Boto/botocore helpers"""
import six
from scrapy.exceptions import NotConfigured
def is_botocore():
try:
import botocore
return True
except ImportError:
if six.PY2:
try:
import boto
return False
except ImportError:
raise NotConfigured('missing botocore or boto library')
else:
raise NotConfigured('missing botocore library')

16
scrapy/utils/test.py
View File

@ -5,9 +5,11 @@ This module contains some assorted functions used in tests
import os
from importlib import import_module
import six
from twisted.trial.unittest import SkipTest
from scrapy.exceptions import NotConfigured
from scrapy.utils.boto import is_botocore
def assert_aws_environ():
"""Asserts the current environment is suitable for running AWS testsi.
@ -19,15 +21,9 @@ def assert_aws_environ():
def skip_if_no_boto():
try:
import botocore
except ImportError:
if six.PY2:
try:
import boto
except ImportError:
raise SkipTest('missing botocore or boto library')
else:
raise SkipTest('missing botocore library')
is_botocore()
except NotConfigured as e:
raise SkipTest(e.message)
def get_crawler(spidercls=None, settings_dict=None):
"""Return an unconfigured Crawler object. If settings_dict is given, it

24
tests/test_feedexport.py
View File

@ -22,6 +22,7 @@ from scrapy.extensions.feedexport import (
)
from scrapy.utils.test import assert_aws_environ
from scrapy.utils.python import to_native_str
from scrapy.utils.boto import is_botocore
class FileFeedStorageTest(unittest.TestCase):
@ -95,17 +96,30 @@ class S3FeedStorageTest(unittest.TestCase):
uri = os.environ.get('FEEDTEST_S3_URI')
if not uri:
raise unittest.SkipTest("No S3 URI available for testing")
from boto import connect_s3
storage = S3FeedStorage(uri)
verifyObject(IFeedStorage, storage)
file = storage.open(scrapy.Spider("default"))
file.write("content")
yield storage.store(file)
u = urlparse(uri)
bucket = connect_s3().get_bucket(u.hostname, validate=False)
key = bucket.get_key(u.path)
self.assertEqual(key.get_contents_as_string(), "content")
bucket.delete_key(u.path)
content = self._get_content_and_delete(u.hostname, u.path[1:])
self.assertEqual(content, "content")
def _get_content_and_delete(self, bucket, path):
if is_botocore():
import botocore.session
session = botocore.session.get_session()
client = session.create_client('s3')
key = client.get_object(Bucket=bucket, Key=path)
content = key['Body'].read()
client.delete_object(Bucket=bucket, Key=path)
else:
from boto import connect_s3
bucket = connect_s3().get_bucket(bucket, validate=False)
key = bucket.get_key(path)
content = key.get_contents_as_string()
bucket.delete_key(path)
return content
class StdoutFeedStorageTest(unittest.TestCase):