diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index cf70b8aca..7e32da6f4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -164,6 +164,11 @@ The feeds are stored in a FTP server. * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv`` * Required external libraries: none +FTP supports two different connection modes: [active or passive]( +https://stackoverflow.com/a/1699163). Scrapy uses the passive connection mode +by default. To use the active connection mode instead, set the +:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``. + .. _topics-feed-storage-s3: S3 @@ -209,6 +214,7 @@ These are the settings used for configuring the feed exports: * :setting:`FEED_URI` (mandatory) * :setting:`FEED_FORMAT` * :setting:`FEED_STORAGES` + * :setting:`FEED_STORAGE_FTP_ACTIVE` * :setting:`FEED_STORAGE_S3_ACL` * :setting:`FEED_EXPORTERS` * :setting:`FEED_STORE_EMPTY` @@ -307,6 +313,16 @@ Default: ``{}`` A dict containing additional feed storage backends supported by your project. The keys are URI schemes and the values are paths to storage classes. +.. setting:: FEED_STORAGE_FTP_ACTIVE + +FEED_STORAGE_FTP_ACTIVE +----------------------- + +Default: ``False`` + +Whether to use [active mode](https://stackoverflow.com/a/1699163) when exporting feeds +to a FTP server. + .. setting:: FEED_STORAGE_S3_ACL FEED_STORAGE_S3_ACL diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 39d05de49..d35551fdd 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -157,19 +157,29 @@ class S3FeedStorage(BlockingFeedStorage): class FTPFeedStorage(BlockingFeedStorage): - def __init__(self, uri): + def __init__(self, uri, use_active_mode=False): u = urlparse(uri) self.host = u.hostname self.port = int(u.port or '21') self.username = u.username self.password = u.password self.path = u.path + self.use_active_mode = use_active_mode + + @classmethod + def from_crawler(cls, crawler, uri): + return cls( + uri=uri, + use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE') + ) def _store_in_thread(self, file): file.seek(0) ftp = FTP() ftp.connect(self.host, self.port) ftp.login(self.username, self.password) + if self.use_active_mode: + ftp.set_pasv(False) dirname, filename = posixpath.split(self.path) ftp_makedirs_cwd(ftp, dirname) ftp.storbinary('STOR %s' % filename, file) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 10b6cf9bc..d17eb3125 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -158,6 +158,7 @@ FEED_EXPORTERS_BASE = { } FEED_EXPORT_INDENT = 0 +FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_S3_ACL = '' FILES_STORE_S3_ACL = 'private' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 14bb72170..c5063253a 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -71,6 +71,13 @@ class FileFeedStorageTest(unittest.TestCase): class FTPFeedStorageTest(unittest.TestCase): + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = 'test_spider' + crawler = get_crawler(settings_dict=settings) + spider = TestSpider.from_crawler(crawler) + return spider + def test_store(self): uri = os.environ.get('FEEDTEST_FTP_URI') path = os.environ.get('FEEDTEST_FTP_PATH') @@ -80,9 +87,20 @@ class FTPFeedStorageTest(unittest.TestCase): verifyObject(IFeedStorage, st) return self._assert_stores(st, path) + def test_store_active_mode(self): + uri = os.environ.get('FEEDTEST_FTP_URI') + path = os.environ.get('FEEDTEST_FTP_PATH') + if not (uri and path): + raise unittest.SkipTest("No FTP server available for testing") + use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True} + crawler = get_crawler(settings_dict=use_active_mode) + st = FTPFeedStorage.from_crawler(crawler, uri) + verifyObject(IFeedStorage, st) + return self._assert_stores(st, path) + @defer.inlineCallbacks def _assert_stores(self, storage, path): - spider = scrapy.Spider("default") + spider = self.get_test_spider() file = storage.open(spider) file.write(b"content") yield storage.store(file)