diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index 53f845c22..8ab028c9e 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -11,6 +11,8 @@ from urlparse import urlparse from ftplib import FTP from shutil import copyfileobj +from zope.interface import Interface, implements + from twisted.internet import defer, threads from scrapy import log, signals from scrapy.xlib.pydispatch import dispatcher @@ -20,8 +22,21 @@ from scrapy.utils.misc import load_object from scrapy.utils.url import file_uri_to_path from scrapy.conf import settings + +class IFeedStorage(Interface): + """Interface that all Feed Storages must implement""" + + def __init__(uri): + """Initialize the storage with the parameters given in the URI""" + + def store(file, spider): + """Store the given file stream""" + + class BlockingFeedStorage(object): + implements(IFeedStorage) + def store(self, file, spider): return threads.deferToThread(self._store_in_thread, file, spider) @@ -31,11 +46,13 @@ class BlockingFeedStorage(object): class StdoutFeedStorage(object): - def __init__(self, uri): - pass + implements(IFeedStorage) + + def __init__(self, uri, _stdout=sys.stdout): + self._stdout = _stdout def store(self, file, spider): - copyfileobj(file, sys.stdout) + copyfileobj(file, self._stdout) class FileFeedStorage(BlockingFeedStorage): diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index ba2b586a6..fa3977533 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -1,11 +1,12 @@ import os, urlparse +from zope.interface.verify import verifyObject from twisted.trial import unittest from twisted.internet import defer from cStringIO import StringIO from scrapy.spider import BaseSpider -from scrapy.contrib.feedexport import FileFeedStorage, FTPFeedStorage, S3FeedStorage +from scrapy.contrib.feedexport import IFeedStorage, FileFeedStorage, FTPFeedStorage, S3FeedStorage, StdoutFeedStorage from scrapy.utils.url import path_to_file_uri from scrapy.utils.test import assert_aws_environ @@ -20,7 +21,6 @@ class FeedStorageTest(unittest.TestCase): yield storage.store(StringIO("new content"), BaseSpider("default")) self.failUnlessEqual(open(path).read(), "new content") - class FileFeedStorageTest(FeedStorageTest): def test_store_file_uri(self): @@ -42,6 +42,11 @@ class FileFeedStorageTest(FeedStorageTest): path = self.mktemp() return self._assert_stores(FileFeedStorage(path), path) + def test_interface(self): + path = self.mktemp() + st = FileFeedStorage(path) + verifyObject(IFeedStorage, st) + class FTPFeedStorageTest(FeedStorageTest): @@ -50,7 +55,9 @@ class FTPFeedStorageTest(FeedStorageTest): path = os.environ.get('FEEDTEST_FTP_PATH') if not (uri and path): raise unittest.SkipTest("No FTP server available for testing") - return self._assert_stores(FTPFeedStorage(uri), path) + st = FTPFeedStorage(uri) + verifyObject(IFeedStorage, st) + return self._assert_stores(st, path) class S3FeedStorageTest(unittest.TestCase): @@ -63,8 +70,17 @@ class S3FeedStorageTest(unittest.TestCase): raise unittest.SkipTest("No S3 URI available for testing") from boto import connect_s3 storage = S3FeedStorage(uri) + verifyObject(IFeedStorage, storage) yield storage.store(StringIO("content"), BaseSpider("default")) u = urlparse.urlparse(uri) key = connect_s3().get_bucket(u.hostname, validate=False).get_key(u.path) self.failUnlessEqual(key.get_contents_as_string(), "content") +class StdoutFeedStorageTest(FeedStorageTest): + + @defer.inlineCallbacks + def test_store(self): + out = StringIO() + storage = StdoutFeedStorage('stdout:', _stdout=out) + yield storage.store(StringIO("content"), BaseSpider("default")) + self.assertEqual(out.getvalue(), "content")