From df0e1f005f8296106f7eb782ff17d974220f3241 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 1 Sep 2009 08:56:54 -0300 Subject: [PATCH] exporters doc: fixed example and some typos --- docs/topics/exporters.rst | 101 +++++++++++++++++----------- scrapy/conf/default_settings.py | 8 ++- scrapy/spider/__init__.py | 5 +- scrapy/spider/manager.py | 114 -------------------------------- 4 files changed, 69 insertions(+), 159 deletions(-) delete mode 100644 scrapy/spider/manager.py diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 8198c658c..f0ccc4c4e 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -7,12 +7,12 @@ Item Exporters .. module:: scrapy.contrib.exporter :synopsis: Item Exporters -Once you have scraped your Items, one of the most common tasks to perform on -those items is to export them, to use the data in some other application. That -is, after all, the whole purpose of the scraping process. +Once you have scraped your Items, you often want to persist or export those +items, to use the data in some other application. That is, after all, the whole +purpose of the scraping process. -To help in this purpose Scrapy provides a collectioon of Item Exporters for -different output formats, such as XML, CSV or JSON. +For this purpose Scrapy provides a collection of Item Exporters for different +output formats, such as XML, CSV or JSON. Using Item Exporters ==================== @@ -24,16 +24,21 @@ work, or need more custom functionality (not covered by the :ref:`File Export Pipeline `) continue reading below. In order to use a Item Exporter, you must instantiate it with its required -args. Different exporters require different args, so check each exporter +args. Each Item Exporter requires different arguments, so check each exporter documentation to be sure, in :ref:`topics-exporters-reference`. After you have -instantiated you exporter, you call the method -:meth:`~BaseItemExporter.start_exporting` in order to initialize the exporting -proces, then you call :meth:`~BaseItemExporter.export_item` method for each -item you want to export, and finally call -:meth:`~BaseItemExporter.finish_exporting` to finalize the exporting process. +instantiated you exporter, you have to: -Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline -`:: +1. call the method :meth:`~BaseItemExporter.start_exporting` in order to +signal the beginning of the exporting process + +2. call the :meth:`~BaseItemExporter.export_item` method for each item you want +to export + +3. and finally call the :meth:`~BaseItemExporter.finish_exporting` to signal +the end of the exporting process + +Here you can see an :doc:`Item Pipeline ` which uses an Item +Exporter to export scraped items to different files, one per spider:: from scrapy.xlib.pydispatch import dispatcher from scrapy.contrib.exporter import XmlItemExporter @@ -43,15 +48,18 @@ Here you can see a typical Item Exporter usage in an :ref:`Item Pipeline def __init__(self): dispatcher.connect(self.domain_opened, signals.domain_opened) dispatcher.connect(self.domain_closed, signals.domain_closed) + self.files = {} def domain_opened(self, domain): - self.file = open('%s_products.xml' % domain) - self.exporter = XmlItemExporter(self.file) + file = open('%s_products.xml' % domain, 'w+b') + self.files[domain] = file + self.exporter = XmlItemExporter(file) self.exporter.start_exporting() def domain_closed(self, domain): self.exporter.finish_exporting() - self.file.close() + file = self.files.pop(domain) + file.close() def process_item(self, domain, item): self.exporter.export_item(item) @@ -67,11 +75,11 @@ By default the field values are passed unmodified to the underlying serialization library, and the decision of how to serialize them is delegated to each particular serialization library. -However, you can customize how each field value is serialized, prior to passing -it to the serialization library, if the exporter supports it. +However, you can customize how each field value is serialized *before it is +passed to the serialization library*. -There are ways to customize how a field will be serialized, which are described -next. +There are two ways to customize how a field will be serialized, which are +described next. .. _topics-exporters-serializers: @@ -119,8 +127,8 @@ Example:: Built-in Item Exporters reference ================================= -For the examples shown in the following exporters we always assume we export -these two items:: +Here is a list of the Item Exporters bundled with Scrapy. Some of them contain +output examples, which assume you're exporting these two items:: Item(name='Color TV', price='1200') Item(name='DVD player', price='200') @@ -136,13 +144,12 @@ BaseItemExporter encoding to use. These features can be configured through the constructor arguments which - populate their respective attributes: :attr:`fields_to_export`, + populate their respective instance attributes: :attr:`fields_to_export`, :attr:`export_empty_fields`, :attr:`encoding`. .. method:: export_item(item) - Exports the item to the specific exporter format. This method must be - implemented in subclasses. + Exports the given item. This method must be implemented in subclasses. .. method:: serialize_field(field, name, value) @@ -166,14 +173,17 @@ BaseItemExporter .. method:: start_exporting() - Makes the exporter initialize the export process, in here exporters may - output information required by the exporter's format. + Signal the beginning of the exporting process. Some exporters may use + this to generate some required header (for example, the + :class:`XmlItemExporter`). You must call this method before exporting any + items. .. method:: finish_exporting() - You must call it when there are no more items to export, so the exporter - can close the serialization output, for those formats that require it - (like XML). + Signal the end of the exporting process. Some exporters may use this to + generate some required footer (for example, the + :class:`XmlItemExporter`). You must always call this method after you + have no more items to export. .. attribute:: fields_to_export @@ -186,7 +196,8 @@ BaseItemExporter .. attribute:: export_empty_fields Whether to include empty/unpopulated item fields in the exported data. - Defaults to ``False``. + Defaults to ``False``. Some exporters (like :class:`CsvItemExporter`) + ignore this attribute and always export all empty fields. .. attribute:: encoding @@ -204,6 +215,8 @@ XmlItemExporter Exports Items in XML format to the specified file object. + :param file: the file-like object to use for exporting the data. + :param root_element: The name of root element in the exported XML. :type root_element: str @@ -238,6 +251,8 @@ CsvItemExporter CSV columns and their order. The :attr:`export_empty_fields` attribute has no effect on this exporter. + :param file: the file-like object to use for exporting the data. + :param include_headers_line: If enabled, makes the exporter output a header line with the field names taken from :attr:`BaseItemExporter.fields_to_export` so that attribute must also be @@ -245,9 +260,9 @@ CsvItemExporter :type include_headers_line: boolean The additional keyword arguments of this constructor are passed to the - :class:`BaseItemExporter` constructor, and then to the `csv.writer`_ - constructor, so you can use any `csv.writer` constructor argument to - customize this exporter. + :class:`BaseItemExporter` constructor, and the leftover arguments to the + `csv.writer`_ constructor, so you can use any `csv.writer` constructor + argument to customize this exporter. A typical output of this exporter would be:: @@ -262,7 +277,9 @@ PickleItemExporter .. class:: PickleItemExporter(file, protocol=0, \**kwargs) Exports Items in pickle format to the given file-like object. - + + :param file: the file-like object to use for exporting the data. + :param protocol: The pickle protocol to use. :type protocol: int @@ -271,7 +288,7 @@ PickleItemExporter The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor. - This isn't a human readable format, so no output examples are provided. + Pickle isn't a human readable format, so no output examples are provided. .. _pickle module documentation: http://docs.python.org/library/pickle.html @@ -282,6 +299,8 @@ PprintItemExporter Exports Items in pretty print format to the specified file object. + :param file: the file-like object to use for exporting the data. + The additional keyword arguments of this constructor are passed to the :class:`BaseItemExporter` constructor. @@ -302,11 +321,13 @@ JsonLinesItemExporter Exports Items in JSON format to the specified file-like object, writing one JSON-encoded item per line. The additional constructor arguments are passed - to the :class:`BaseItemExporter` constructor, and to the `JSONEncoder`_ - constructor, so you can use any `JSONEncoder`_ constructor argument to - customize the exporter. + to the :class:`BaseItemExporter` constructor, and the leftover arguments to + the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_ + constructor argument to customize this exporter. - The default output of this exporter would be:: + :param file: the file-like object to use for exporting the data. + + A typical output of this exporter would be:: {"name": "Color TV", "price": "1200"} {"name": "DVD player", "price": "200"} diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index b837c0483..d14d4958f 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -146,9 +146,7 @@ SCHEDULER_MIDDLEWARES_BASE = { SCHEDULER_ORDER = 'BFO' # available orders: BFO (default), DFO -SPIDER_MODULES = [] - -SPIDERPROFILER_ENABLED = False +SPIDER_MANAGER_CLASS = 'scrapy.contrib.spidermanager.TwistedPluginSpiderManager' SPIDER_MIDDLEWARES = {} @@ -165,6 +163,10 @@ SPIDER_MIDDLEWARES_BASE = { # Spider side } +SPIDER_MODULES = [] + +SPIDERPROFILER_ENABLED = False + STATS_CLASS = 'scrapy.stats.collector.MemoryStatsCollector' STATS_ENABLED = True STATS_DUMP = False diff --git a/scrapy/spider/__init__.py b/scrapy/spider/__init__.py index 2a88816de..875bec845 100644 --- a/scrapy/spider/__init__.py +++ b/scrapy/spider/__init__.py @@ -1,4 +1,5 @@ from scrapy.spider.models import BaseSpider -from scrapy.spider.manager import SpiderManager +from scrapy.utils.misc import load_object +from scrapy.conf import settings -spiders = SpiderManager() +spiders = load_object(settings['SPIDER_MANAGER_CLASS'])() diff --git a/scrapy/spider/manager.py b/scrapy/spider/manager.py deleted file mode 100644 index 913450f84..000000000 --- a/scrapy/spider/manager.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -SpiderManager is the class which locates and manages all website-specific -spiders -""" -import sys -import urlparse - -from twisted.plugin import getCache - -from scrapy.spider.models import ISpider -from scrapy import log -from scrapy.conf import settings -from scrapy.utils.url import url_is_from_spider -from scrapy.utils.misc import load_object - -class SpiderManager(object): - """Spider locator and manager""" - - def __init__(self): - self.loaded = False - self.default_domain = None - self.force_domain = None - self.spider_modules = None - - def fromdomain(self, domain_name): - return self.asdict().get(domain_name) - - def fromurl(self, url): - self._load_on_demand() - if self.force_domain: - return self.asdict().get(self.force_domain) - domain = urlparse.urlparse(url).hostname - domain = str(domain).replace('www.', '') - if domain: - if domain in self.asdict(): # try first locating by domain - return self.asdict()[domain] - else: # else search spider by spider - plist = self.asdict().values() - for p in plist: - if url_is_from_spider(url, p): - return p - spider = self.asdict().get(self.default_domain) - if not spider: # create a custom spider - spiderclassname = settings.get('DEFAULT_SPIDER') - if spiderclassname: - spider = load_object(spiderclassname)(domain) - self.add_spider(spider) - - return spider - - def asdict(self): - self._load_on_demand() - return self._spiders - - def _load_on_demand(self): - if not self.loaded: - self.load() - - def load(self, spider_modules=None): - if spider_modules is None: - spider_modules = settings.getlist('SPIDER_MODULES') - self.spider_modules = spider_modules - self._invaliddict = {} - self._spiders = {} - - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for module in modules: - for spider in self._getspiders(ISpider, module): - self.add_spider(spider) - self.loaded = True - - def add_spider(self, spider): - try: - ISpider.validateInvariants(spider) - self._spiders[spider.domain_name] = spider - except Exception, e: - self._invaliddict[spider.domain_name] = spider - # we can't use the log module here because it may not be available yet - print "WARNING: Could not load spider %s: %s" % (spider, e) - - def reload(self, spider_modules=None, skip_domains=None): - """Reload spiders by trying to discover any spiders added under the - spiders module/packages, removes any spiders removed. - - If skip_domains is passed those spiders won't be reloaded. - """ - skip_domains = set(skip_domains or []) - modules = [__import__(m, {}, {}, ['']) for m in self.spider_modules] - for m in modules: - reload(m) - reloaded = 0 - pdict = self.asdict() - for domain, spider in pdict.iteritems(): - if not domain in skip_domains: - reload(sys.modules[spider.__module__]) - reloaded += 1 - self.load(spider_modules=spider_modules) # second call to update spider instances - log.msg("Reloaded %d/%d scrapy spiders" % (reloaded, len(pdict)), level=log.DEBUG) - - def _getspiders(self, interface, package): - """This is an override of twisted.plugin.getPlugin, because we're - interested in catching exceptions thrown when loading spiders such as - KeyboardInterrupt - """ - try: - allDropins = getCache(package) - for dropin in allDropins.itervalues(): - for plugin in dropin.plugins: - adapted = interface(plugin, None) - if adapted is not None: - yield adapted - except KeyboardInterrupt: - sys.stderr.write("Interrupted while loading Scrapy spiders\n") - sys.exit(2)