RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix Flake8-reported issues

This commit is contained in:
Adrián Chaves 2020-02-12 17:38:06 +01:00
parent 73e88d036c
commit e1be078eaa
2 changed files with 16 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/utils/test.py
View File

@ -120,12 +120,14 @@ def assert_samelines(testcase, text1, text2, msg=None):
"""
testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
def get_from_asyncio_queue(value):
q = asyncio.Queue()
getter = q.get()
q.put_nowait(value)
return getter
def mock_google_cloud_storage():
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
classes and set their proper return values.

27
tests/test_feedexport.py
View File

@ -25,9 +25,13 @@ from scrapy.extensions.feedexport import (
IFeedStorage, FileFeedStorage, FTPFeedStorage, GCSFeedStorage,
S3FeedStorage, StdoutFeedStorage,
BlockingFeedStorage)
from scrapy.utils.test import (assert_aws_environ, get_s3_content_and_delete,
get_crawler, mock_google_cloud_storage)
from scrapy.utils.python import to_unicode
from scrapy.utils.test import (
assert_aws_environ,
get_s3_content_and_delete,
get_crawler,
mock_google_cloud_storage,
)
class FileFeedStorageTest(unittest.TestCase):
@ -362,15 +366,13 @@ class S3FeedStorageTest(unittest.TestCase):
class GCSFeedStorageTest(unittest.TestCase):
@mock.patch('scrapy.conf.settings',
new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': None }, create=True)
def test_parse_settings(self):
try:
from google.cloud.storage import Client
from google.cloud.storage import Client # noqa
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead' }
settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': 'publicRead'}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
assert storage.project_id == '123'
@ -378,20 +380,18 @@ class GCSFeedStorageTest(unittest.TestCase):
assert storage.bucket_name == 'mybucket'
assert storage.blob_name == 'export.csv'
@mock.patch('scrapy.conf.settings',
new={'GCS_PROJECT_ID': 'conf_id', 'FEED_STORAGE_GCS_ACL': '' }, create=True)
def test_parse_empty_acl(self):
try:
from google.cloud.storage import Client
from google.cloud.storage import Client # noqa
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': '' }
settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': ''}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
assert storage.acl is None
settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None }
settings = {'GCS_PROJECT_ID': '123', 'FEED_STORAGE_GCS_ACL': None}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, 'gs://mybucket/export.csv')
assert storage.acl is None
@ -399,7 +399,7 @@ class GCSFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def test_store(self):
try:
from google.cloud.storage import Client
from google.cloud.storage import Client # noqa
except ImportError:
raise unittest.SkipTest("GCSFeedStorage requires google-cloud-storage")
@ -407,7 +407,7 @@ class GCSFeedStorageTest(unittest.TestCase):
project_id = 'myproject-123'
acl = 'publicRead'
(client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
with mock.patch('google.cloud.storage.Client') as m:
with mock.patch('google.cloud.storage.Client') as m:
m.return_value = client_mock
f = mock.Mock()
@ -420,6 +420,7 @@ class GCSFeedStorageTest(unittest.TestCase):
bucket_mock.blob.assert_called_once_with('export.csv')
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
class StdoutFeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks