added FEED_TEMPDIR to settings

docs/topics/settings.rst

@@ -178,6 +178,15 @@ Default: ``None``
 The AWS secret key used by code that requires access to `Amazon Web services`_,
 such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
 
+.. setting:: FEED_TEMPDIR
+
+FEED_TEMPDIR
+------------
+
+The Feed Temp dir allows you to set a custom folder to save crawler
+temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
+:ref:`Amazon S3 <topics-feed-storage-s3>`.
+
 .. setting:: BOT_NAME
 
 BOT_NAME

scrapy/extensions/feedexport.py

@@ -8,7 +8,7 @@ import os
 import sys
 import logging
 import posixpath
-from tempfile import TemporaryFile
+from tempfile import NamedTemporaryFile
 from datetime import datetime
 import six
 from six.moves.urllib.parse import urlparse
@@ -47,7 +47,11 @@ class IFeedStorage(Interface):
 class BlockingFeedStorage(object):
 
     def open(self, spider):
-        return TemporaryFile(prefix='feed-')
+        path = spider.crawler.settings['FEED_TEMPDIR']
+        if path and not os.path.isdir(path):
+            raise OSError('Not a Directory: ' + str(path))
+
+        return NamedTemporaryFile(prefix='feed-', dir=path)
 
     def store(self, file):
         return threads.deferToThread(self._store_in_thread, file)

scrapy/settings/default_settings.py

@@ -134,6 +134,7 @@ EXTENSIONS_BASE = {
     'scrapy.extensions.throttle.AutoThrottle': 0,
 }
 
+FEED_TEMPDIR = None
 FEED_URI = None
 FEED_URI_PARAMS = None  # a function to extend uri arguments
 FEED_FORMAT = 'jsonlines'

tests/test_feedexport.py

@@ -18,9 +18,9 @@ from w3lib.url import path_to_file_uri
 import scrapy
 from scrapy.extensions.feedexport import (
     IFeedStorage, FileFeedStorage, FTPFeedStorage,
-    S3FeedStorage, StdoutFeedStorage
-)
-from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
+    S3FeedStorage, StdoutFeedStorage,
+    BlockingFeedStorage)
+from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
 from scrapy.utils.python import to_native_str
 
 
@@ -87,6 +87,41 @@ class FTPFeedStorageTest(unittest.TestCase):
             self.assertEqual(fp.read(), b"new content")
 
 
+class BlockingFeedStorageTest(unittest.TestCase):
+
+    def get_test_spider(self, settings=None):
+        class TestSpider(scrapy.Spider):
+            name = 'test_spider'
+        crawler = get_crawler(settings_dict=settings)
+        spider = TestSpider.from_crawler(crawler)
+        return spider
+
+    def test_default_temp_dir(self):
+        b = BlockingFeedStorage()
+
+        tmp = b.open(self.get_test_spider())
+        tmp_path = os.path.dirname(tmp.name)
+        self.assertEqual(tmp_path, tempfile.gettempdir())
+
+    def test_temp_file(self):
+        b = BlockingFeedStorage()
+
+        tests_path = os.path.dirname(os.path.abspath(__file__))
+        spider = self.get_test_spider({'FEED_TEMPDIR': tests_path})
+        tmp = b.open(spider)
+        tmp_path = os.path.dirname(tmp.name)
+        self.assertEqual(tmp_path, tests_path)
+
+    def test_invalid_folder(self):
+        b = BlockingFeedStorage()
+
+        tests_path = os.path.dirname(os.path.abspath(__file__))
+        invalid_path = os.path.join(tests_path, 'invalid_path')
+        spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path})
+
+        self.assertRaises(OSError, b.open, spider=spider)
+
+
 class S3FeedStorageTest(unittest.TestCase):
 
     @defer.inlineCallbacks