diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e42dc6370..1bf428edb 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -178,6 +178,15 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: FEED_TEMPDIR + +FEED_TEMPDIR +------------ + +The Feed Temp dir allows you to set a custom folder to save crawler +temporary files before uploading with :ref:`FTP feed storage ` and +:ref:`Amazon S3 `. + .. setting:: BOT_NAME BOT_NAME diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 3dab2d77e..5d90ef738 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -8,7 +8,7 @@ import os import sys import logging import posixpath -from tempfile import TemporaryFile +from tempfile import NamedTemporaryFile from datetime import datetime import six from six.moves.urllib.parse import urlparse @@ -47,7 +47,11 @@ class IFeedStorage(Interface): class BlockingFeedStorage(object): def open(self, spider): - return TemporaryFile(prefix='feed-') + path = spider.crawler.settings['FEED_TEMPDIR'] + if path and not os.path.isdir(path): + raise OSError('Not a Directory: ' + str(path)) + + return NamedTemporaryFile(prefix='feed-', dir=path) def store(self, file): return threads.deferToThread(self._store_in_thread, file) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d449c4891..b9d01e155 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -134,6 +134,7 @@ EXTENSIONS_BASE = { 'scrapy.extensions.throttle.AutoThrottle': 0, } +FEED_TEMPDIR = None FEED_URI = None FEED_URI_PARAMS = None # a function to extend uri arguments FEED_FORMAT = 'jsonlines' diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index c76d26b57..05f19d33e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,9 +18,9 @@ from w3lib.url import path_to_file_uri import scrapy from scrapy.extensions.feedexport import ( IFeedStorage, FileFeedStorage, FTPFeedStorage, - S3FeedStorage, StdoutFeedStorage -) -from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete + S3FeedStorage, StdoutFeedStorage, + BlockingFeedStorage) +from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler from scrapy.utils.python import to_native_str @@ -87,6 +87,41 @@ class FTPFeedStorageTest(unittest.TestCase): self.assertEqual(fp.read(), b"new content") +class BlockingFeedStorageTest(unittest.TestCase): + + def get_test_spider(self, settings=None): + class TestSpider(scrapy.Spider): + name = 'test_spider' + crawler = get_crawler(settings_dict=settings) + spider = TestSpider.from_crawler(crawler) + return spider + + def test_default_temp_dir(self): + b = BlockingFeedStorage() + + tmp = b.open(self.get_test_spider()) + tmp_path = os.path.dirname(tmp.name) + self.assertEqual(tmp_path, tempfile.gettempdir()) + + def test_temp_file(self): + b = BlockingFeedStorage() + + tests_path = os.path.dirname(os.path.abspath(__file__)) + spider = self.get_test_spider({'FEED_TEMPDIR': tests_path}) + tmp = b.open(spider) + tmp_path = os.path.dirname(tmp.name) + self.assertEqual(tmp_path, tests_path) + + def test_invalid_folder(self): + b = BlockingFeedStorage() + + tests_path = os.path.dirname(os.path.abspath(__file__)) + invalid_path = os.path.join(tests_path, 'invalid_path') + spider = self.get_test_spider({'FEED_TEMPDIR': invalid_path}) + + self.assertRaises(OSError, b.open, spider=spider) + + class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks