Add a setting to use active mode in FTPFeedStorage (#3829)

Add a setting to use active mode in FTPFeedStorage

FTP servers can be configured in active and passive mode, by default
Python and Scrapy use passive mode and there was no way to use active.

This commit adds a setting FEED_STORAGE_FTP_ACTIVE to allow a feed
to be exported to a FTP server configured in active mode.

docs/topics/feed-exports.rst

@@ -164,6 +164,11 @@ The feeds are stored in a FTP server.
  * Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
  * Required external libraries: none
 
+FTP supports two different connection modes: [active or passive](
+https://stackoverflow.com/a/1699163). Scrapy uses the passive connection mode
+by default. To use the active connection mode instead, set the
+:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -209,6 +214,7 @@ These are the settings used for configuring the feed exports:
  * :setting:`FEED_URI` (mandatory)
  * :setting:`FEED_FORMAT`
  * :setting:`FEED_STORAGES`
+ * :setting:`FEED_STORAGE_FTP_ACTIVE`
  * :setting:`FEED_STORAGE_S3_ACL`
  * :setting:`FEED_EXPORTERS`
  * :setting:`FEED_STORE_EMPTY`
@@ -307,6 +313,16 @@ Default: ``{}``
 A dict containing additional feed storage backends supported by your project.
 The keys are URI schemes and the values are paths to storage classes.
 
+.. setting:: FEED_STORAGE_FTP_ACTIVE
+
+FEED_STORAGE_FTP_ACTIVE
+-----------------------
+
+Default: ``False``
+
+Whether to use [active mode](https://stackoverflow.com/a/1699163) when exporting feeds
+to a FTP server.
+
 .. setting:: FEED_STORAGE_S3_ACL
 
 FEED_STORAGE_S3_ACL

scrapy/extensions/feedexport.py

@@ -157,19 +157,29 @@ class S3FeedStorage(BlockingFeedStorage):
 
 class FTPFeedStorage(BlockingFeedStorage):
 
-    def __init__(self, uri):
+    def __init__(self, uri, use_active_mode=False):
         u = urlparse(uri)
         self.host = u.hostname
         self.port = int(u.port or '21')
         self.username = u.username
         self.password = u.password
         self.path = u.path
+        self.use_active_mode = use_active_mode
+
+    @classmethod
+    def from_crawler(cls, crawler, uri):
+        return cls(
+            uri=uri,
+            use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
+        )
 
     def _store_in_thread(self, file):
         file.seek(0)
         ftp = FTP()
         ftp.connect(self.host, self.port)
         ftp.login(self.username, self.password)
+        if self.use_active_mode:
+            ftp.set_pasv(False)
         dirname, filename = posixpath.split(self.path)
         ftp_makedirs_cwd(ftp, dirname)
         ftp.storbinary('STOR %s' % filename, file)

scrapy/settings/default_settings.py

@@ -158,6 +158,7 @@ FEED_EXPORTERS_BASE = {
 }
 FEED_EXPORT_INDENT = 0
 
+FEED_STORAGE_FTP_ACTIVE = False
 FEED_STORAGE_S3_ACL = ''
 
 FILES_STORE_S3_ACL = 'private'

tests/test_feedexport.py

@@ -71,6 +71,13 @@ class FileFeedStorageTest(unittest.TestCase):
 
 class FTPFeedStorageTest(unittest.TestCase):
 
+    def get_test_spider(self, settings=None):
+        class TestSpider(scrapy.Spider):
+            name = 'test_spider'
+        crawler = get_crawler(settings_dict=settings)
+        spider = TestSpider.from_crawler(crawler)
+        return spider
+
     def test_store(self):
         uri = os.environ.get('FEEDTEST_FTP_URI')
         path = os.environ.get('FEEDTEST_FTP_PATH')
@@ -80,9 +87,20 @@ class FTPFeedStorageTest(unittest.TestCase):
         verifyObject(IFeedStorage, st)
         return self._assert_stores(st, path)
 
+    def test_store_active_mode(self):
+        uri = os.environ.get('FEEDTEST_FTP_URI')
+        path = os.environ.get('FEEDTEST_FTP_PATH')
+        if not (uri and path):
+            raise unittest.SkipTest("No FTP server available for testing")
+        use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True}
+        crawler = get_crawler(settings_dict=use_active_mode)
+        st = FTPFeedStorage.from_crawler(crawler, uri)
+        verifyObject(IFeedStorage, st)
+        return self._assert_stores(st, path)
+
     @defer.inlineCallbacks
     def _assert_stores(self, storage, path):
-        spider = scrapy.Spider("default")
+        spider = self.get_test_spider()
         file = storage.open(spider)
         file.write(b"content")
         yield storage.store(file)