From d4aa72d2bb00d6717b1c83688637d4d1a5b405ff Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 12 Aug 2009 13:50:15 -0300 Subject: [PATCH 1/4] removed obsolete documentation about Robust Scraped Item and Adaptors --- docs/topics/adaptors.rst | 205 --------------------------------------- docs/topics/items.rst | 141 --------------------------- 2 files changed, 346 deletions(-) delete mode 100644 docs/topics/adaptors.rst diff --git a/docs/topics/adaptors.rst b/docs/topics/adaptors.rst deleted file mode 100644 index 9088beb62..000000000 --- a/docs/topics/adaptors.rst +++ /dev/null @@ -1,205 +0,0 @@ -.. _topics-adaptors: - -============================== -Old Item Adaptors (deprecated) -============================== - -.. warning:: - - This documentation is deprecated, and will be superseeded by the new item - adaptors (not yet documented). - -Quick overview -============== - -Scrapy's adaptors are a nice feature attached to :class:`RobustScrapedItem` -that allow you to easily modify (adapt to your needs) any kind of information -you want to put in your items at assignation time. - -The following diagram shows the data flow from the moment you call the -``attribute`` method until the attribute is actually set. - -.. image:: _images/adaptors_diagram.png - -As you can see, adaptor pipelines are executed in tree form; which means that, -for each of the values you pass to the ``attribute`` method, the first adaptor -will be applied. Then, for each of the resulting values of the first adaptor, -the second adaptor will be called, and so on. This process will end up with a -list of adapted values, which may contain zero, one, or many values. - -In case the attribute is a single-valued (this is defined in the item's -``ATTRIBUTES`` dictionary), the first element of this list will be set, unless -you call the ``attribute`` method with the add parameter as True, in which case -the item's method ``_add_single_attributes`` will be called with the -attribute's name, type, and the list of attributes to join as parameters. By -default, this method raises NotImplementedError, so you should override it in -your items in order to join any kind of objects. - -If the attribute is a multivalued, the resulting list will be set to the item -as is, unless you use -again- add=True, in which case the list of -already-existing values (if any) will be extended with the new one.pgq - -Adaptor Pipelines -================= - -.. class:: AdaptorPipe(adaptors=None) - - An instance of this class represents an adaptor pipeline to be set for - adapting a certain item's attribute. It provides some useful methods for - adding/removing adaptors, and takes care of executing them properly. - Usually this class is not used directly, since the items already provide - ways to manage adaptors without having to handle AdaptorPipes. - - :param adaptors: A list of callables to be added as adaptors at - instancing time. - - Methods: - - .. method:: add_adaptor(adaptor, position=None) - - This method is used for adding adaptors to the pipeline given - a certain position. - - :param adaptor: Any callable that works as an adaptor - :param position: An integer meaning the position in which the adaptor - will be inserted. If it's None the adaptor will be appended at - the end of the pipeline. - -Usage -===== - -As it was previously said, in order to use adaptor pipelines you must inherit -your items from the :class:`RobustScrapedItem` class. If you don't know -anything about these items, read the :ref:`topics-items` reference first. - -Once you've created your own item class (inherited from -:class:`RobustScrapedItem`) with the attributes you're going to use, you have -to add adaptor pipelines to each attribute you'd like to adapt data for. For -doing so, RobustScrapedItems provide some useful methods like ``set_adaptors``, -``set_attrib_adaptors``, and more (which are also described in its reference) -so that you don't need to work with :class:`AdaptorPipe` objects directly. - -Adaptors --------- - -Let's now talk a bit about adaptors (singularly), what are them, and how -should they be implemented? - -Adaptors are basically, any callable that receives -a value, modifies it, and returns a new value (or more) so that the next -adaptor goes on with another adapting task (or not). This is done this way to -make the process of modifying information very customizable, and also to make -adaptors reusable, since they are intended to be small functions designed for -simple purposes that can be applied in many different cases. For example, you -could make an adaptor for removing any tags in a text, like this:: - - >>> B_TAG_RE = re.compile(r'') - >>> def remove_b_tags(text): - >>> return B_TAG_RE.sub('', text) - -Then you could easily add this adaptor to a certain attribute's pipeline like -this:: - - >>> item = MyItem() - >>> item.add_adaptor('text', remove_b_tags) - >>> item.attribute('text', u'some random text in bold and some random text in normal font') - >>> item.text - u'some random text in bold and some random text in normal font' - -As you can see, this would make any value that you set to the item through the -``attribute`` method first pass through the ``remove_b_tags`` adaptor, which -would also replace any matching tag with an empty string. - ----- - -But anyway, let's now think of a bit more complicated (and useless) example: -let's say you want to scrape a text, split it into single letters, strip the -vowels, turn the rest to capital letters, and join them again. In this case, -we could use three simple adaptors to process our data, plus a customized -:class:`RobustScrapedItem` for joining single text attributes; let's see an -example:: - - >>> # First of all, we define the item class we're going to use - >>> from string import ascii_letters - >>> from scrapy.contrib.item import RobustScrapedItem - >>> class MyItem(RobustScrapedItem): - >>> ATTRIBUTES = { - >>> 'text': basestring, - >>> } - - >>> def _add_single_attributes(self, attrname, attrtype, attributes): - >>> return ''.join(attributes) - - >>> # Now we'll write the needed adaptors - >>> def to_letters(text): - >>> return tuple(letter for letter in text) - - >>> def is_vowel(letter): - >>> if letter in ascii_letters and letter.lower() not in ('a', 'e', 'i', 'o', 'u'): - >>> return letter - - >>> def to_upper(letter): - >>> return letter.upper() - - >>> # Finally, we'll join all the pieces and see how it works - >>> item = MyItem() - >>> item.set_attrib_adaptors('text', [ - >>> to_letters, - >>> is_vowel, - >>> to_upper, - >>> ]) - -Let's now try with an example text to see what happens:: - - >>> item.attribute('text', 'pi', 'wind', add=True) - >>> item.text - 'PWND' - -More complex adaptors ---------------------- - -Now, after using adaptors a bit, you may find yourself in situations where you need -to use adaptors that receive other parameters from the ``attribute`` method -apart from the value to adapt. - -For example, imagine you have an adaptor that removes certain characters from strings -you provide. Would you make an adaptor for each combination of characters you'd like -to strip? Of course not! - -The way to handle this cases, is to make an adaptor that apart from receiving a value, -as any other adaptor, receives a parameter called ``adaptor_args``. -It's important that the parameter is called this way, since Scrapy finds out whether -an adaptor is able to receive extra parameters or not by making instrospection -and looking for a parameter called this way in the adaptor's parameters list. - -The information this parameter will receive won't be anything else but the same dictionary -of keyword arguments that you pass to the ``attribute`` method when calling it. - -But let's get back to the characters example, how would we implement this? -Quite simmilar to any other adaptor, let's see:: - - def strip_chars(value, adaptor_args): - chars = adaptor_args.get('strip_chars', []) - for char in chars: - value = value.replace(char, '') - return value - -Then, after creating an item and adding the adaptor to one of its pipelines, we could do:: - - >>> item.attribute('text', 'Hi, my name is John', strip_chars=['a', 'i', 'm']) - >>> item.text - 'H, y ne s John' - -Debugging -========= - -While you're coding spiders and adaptors, you usually need to know exactly what -does Scrapy do under the hood with the values you provide. There's a setting -called :setting:``ADAPTORS_DEBUG`` for this purpose that makes Scrapy print -debugging messages each time an adaptors pipeline is run, specifying which -attribute is being adapted data for, the input/output values of each adaptor in -the pipeline, and the input/output of ``_add_single_attributes`` (in some -cases). - -You can enable this setting as any other, either by adding it to your settings -file, or by enabling the environment variable ``SCRAPY_ADAPTORS_DEBUG``. diff --git a/docs/topics/items.rst b/docs/topics/items.rst index c607cf76e..1eee884fa 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -51,144 +51,3 @@ Creating an item and setting its attributes inline:: >>> person ScrapedItem({'age': 23, 'last_name': 'Smith', 'name': 'John'}) -RobustScrapedItems -================== - -.. warning:: - - RobustScapedItems are deprecated and will be replaced by the :ref:`New item - API ` (still in development). - -.. module:: scrapy.contrib.item - :synopsis: Objects for storing scraped data - -.. class:: RobustScrapedItem - - RobustScrapedItems are more complex items (compared to - :class:`ScrapedItem`) and have a few more features available, which - include: - - * Attributes dictionary: items that inherit from RobustScrapedItem are - defined with a dictionary of attributes in the class. This allows the - item to have more logic at the moment of handling and setting attributes - than the :class:`ScrapedItem`. - - * Adaptors: perhaps the most important of the features these items provide. - The adaptors are a system designed for filtering/modifying data before - setting it to the item, that makes cleansing tasks a lot easier. - - * Type checking: RobustScrapedItems come with a built-in type checking - which assures you that no data of the wrong type will get into the items - without raising a warning. - - * Versioning: These items also provide versioning by making a unique hash - for each item based on its attributes values. - - * ItemDeltas: You can subtract two RobustScrapedItems, which allows you to - know the difference between a pair of items. This difference is - represented by a RobustItemDelta object. - -Attributes ----------- - -.. attribute:: RobustScrapedItem.ATTRIBUTES - - This attribute **must** be specified when writing your items, and it's a - dictionary in which the keys are the names of the attributes your item will - have, and their values are the type of those attributes. For multivalued - attributes, you should write the type of the values inside a list, e.g: - ``'numbers': [int]`` - -Methods -------- - -.. method:: RobustScrapedItem.__init__(data=None, adaptor_args=None) - - :param data: Idem as for ScrapedItems - :param adaptor_args: A dictionary of the kind - ``'attribute': [list_of_adaptors]``" for defining adaptors automatically - after instancing the item. - - Constructor of RobustScrapedItem objects. - -.. method:: RobustScrapedItem.attribute(attrname, value, override=False, add=False, ***kwargs) - - Sets the item's ``attrname`` attribute with the given ``value`` filtering - it through the given attribute's adaptor pipeline (if any). - - :param attrname: a string containing the name of the attribute you want - to set. - - :param value: the value you want to assign, which will be adapted by - the corresponding adaptors for the given attribute (if any). - - :param override: if True, makes this method avoid checking if there - was a previous value and sets ``value`` no matter what. - - :param add: if True, tries to concatenate the given ``value`` with the one - already set in the item. For multivalued attributes, this will extend - the list of already-set values, with the new ones. - For single valued attributes, the method _add_single_attributes (which - is explained below) will be called. - - :param kwargs: any extra parameters will be passed in a dictionary to any - adaptor that receives a parameter called ``adaptor_args``. - Check the :ref:`topics-adaptors` topic for more information. - -.. method:: RobustScrapedItem.set_adaptors(adaptors_dict) - - Receives a dict containing a list of adaptors for each desired attribute - (key) and sets each of them as their adaptor pipeline. - -.. method:: RobustScrapedItem.set_attrib_adaptors(attrib, pipe) - - Sets the provided iterable (``pipe``) as the adaptor pipeline for the - given attribute (``attrib``) - -.. method:: RobustScrapedItem.add_adaptor(attrib, adaptor, position=None) - - Adds an adaptor to an already existing (or not) pipeline. - - :param attr: the name of the attribute you're adding adaptors to. - - :param adaptor: a callable to be added to the pipeline. - - :param position: an integer representing the place where to add the adaptor. - If it's ``None``, the adaptor will be appended at the end of the pipeline. - -.. method:: RobustScrapedItem._add_single_attributes(attrname, attrtype, attributes) - - This method is the one to be called whenever a single attribute has to be - joined before storing into an item. That is, - every time you have multiple results at the end of your adaptors pipeline, - and you called the ``attribute`` method with the parameter `add=True`. - - This method is intended to be overriden by you, since by default it - raises an exception. - - :param attrname: the name of the attribute you're setting - :param attrtype: the type of the attribute you're setting - :param attributes: the list of resulting values after the adaptors pipeline - (the one you have to join somehow) - -Examples --------- - -Creating a pretty basic item with a few attributes:: - - from scrapy.contrib.item import RobustScrapedItem - - class MyItem(RobustScrapedItem): - ATTRIBUTES = { - 'name': basestring, - 'size': basestring, - 'colours': [basestring], - } - -Setting some adaptors:: - - -.. note:: - - More RobustScrapedItem examples are about to come. In the meantime, check the :ref:`topics-adaptors` topic to see a few of them. - From 7cbbc3ffb050c18969943713336bb70e894bdf97 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 12 Aug 2009 16:49:05 -0300 Subject: [PATCH 2/4] Renamed Loader to ItemParser (SEP-8 proposal). Documentation and unittests also updated. --HG-- rename : docs/experimental/loaders.rst => docs/experimental/itemparser.rst rename : scrapy/newitem/loader/__init__.py => scrapy/contrib/itemparser/__init__.py rename : scrapy/newitem/loader/expanders.py => scrapy/contrib/itemparser/parsers.py rename : scrapy/tests/test_itemloader.py => scrapy/tests/test_itemparser.py --- docs/experimental/index.rst | 2 +- docs/experimental/itemparser.rst | 526 ++++++++++++++++++++++++++ docs/experimental/loaders.rst | 501 ------------------------ scrapy/contrib/itemparser/__init__.py | 93 +++++ scrapy/contrib/itemparser/common.py | 13 + scrapy/contrib/itemparser/parsers.py | 52 +++ scrapy/newitem/loader/__init__.py | 89 ----- scrapy/newitem/loader/expanders.py | 45 --- scrapy/newitem/loader/reducers.py | 27 -- scrapy/tests/test_itemloader.py | 272 ------------- scrapy/tests/test_itemparser.py | 280 ++++++++++++++ 11 files changed, 965 insertions(+), 935 deletions(-) create mode 100644 docs/experimental/itemparser.rst delete mode 100644 docs/experimental/loaders.rst create mode 100644 scrapy/contrib/itemparser/__init__.py create mode 100644 scrapy/contrib/itemparser/common.py create mode 100644 scrapy/contrib/itemparser/parsers.py delete mode 100644 scrapy/newitem/loader/__init__.py delete mode 100644 scrapy/newitem/loader/expanders.py delete mode 100644 scrapy/newitem/loader/reducers.py delete mode 100644 scrapy/tests/test_itemloader.py create mode 100644 scrapy/tests/test_itemparser.py diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 55468737d..d7eaf261c 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -20,4 +20,4 @@ it's properly merged) . Use at your own risk. :maxdepth: 1 newitems - loaders + itemparser diff --git a/docs/experimental/itemparser.rst b/docs/experimental/itemparser.rst new file mode 100644 index 000000000..160ebe698 --- /dev/null +++ b/docs/experimental/itemparser.rst @@ -0,0 +1,526 @@ +.. _topics-itemparser: + +============ +Item Parsers +============ + +.. module:: scrapy.contrib.itemparser + :synopsis: Item Parser class + +Item Parser provide a convenient mechanism for populating scraped :ref:`Items +`. Even though Items can be populated using their own +dictionary-like API, the Item Parsers provide a much more convenient API for +populating them from a scraping process, by automating some common tasks like +parsing the raw extracted data before assigning it. + +In other words, :ref:`Items ` provide the *container* of +scraped data, while Item Parsers provide the mechanism for *populating* that +container. + +Item Parsers are designed to provide a flexible, efficient and easy mechanism +for extending and overriding different field parsing rules, either by spider, +or by source format (HTML, XML, etc) without becoming a nightmare to maintain. + +Using Item Parsers to populate items +==================================== + +To use an Item Parser, you must first instantiate it. You can either +instantiate it with an Item object or without one, in which case an Item is +automatically instantiated in the Item Parser constructor using the Item class +specified in the :attr:`ItemParser.default_item_class` attribute. + +Then, you start collecting values into the Item Parser, typically using using +:ref:`XPath Selectors `. You can add more than one value to +the same item field, the Item Parser will know how to "join" those values later +using a proper parser function. + +Here is a typical Item Parser usage in a :ref:`Spider `, using +the :ref:`Product item ` declared in the :ref:`Items +chapter `:: + + from scrapy.contrib.itemparser import XPathItemParser + from scrapy.xpath import HtmlXPathSelector + from myproject.items import Product + + def parse(self, response): + p = XPathItemParser(item=Product(), response=response) + p.add_xpath('name', '//div[@class="product_name"]') + p.add_xpath('name', '//div[@class="product_title"]') + p.add_xpath('price', '//p[@id="price"]') + p.add_xpath('stock', '//p[@id="stock"]') + p.add_value('last_updated', 'today') # you can also use literal values + return p.populate_item() + +By quickly looking at that code we can see the ``name`` field is being +extracted from two different XPath locations in the page: + +1. ``//div[@class="product_name"]`` +2. ``//div[@class="product_title"]`` + +In other words, data is being collected by extracting it from two XPath +locations, using the :meth:`~XPathItemParser.add_xpath` method. This is the data +that will be assigned to the ``name`` field later. + +Afterwards, similar calls are used for ``price`` and ``stock`` fields, and +finally the ``last_update`` field is populated directly with a literal value +(``today``) using a different method: :meth:`~ItemParser.add_value`. + +Finally, when all data is collected, the :meth:`ItemParser.populate_item` +method is called which actually populates and returns the item populated with +the data previously extracted and collected with the +:meth:`~XPathItemParser.add_xpath` and :meth:`~ItemParser.add_value` calls. + +.. _topics-itemparser-parsers: + +Input and Output parsers +======================== + +An Item Parser contains one input parser and one output parser for each (item) +field. The input parser processes the extracted data as soon as it's received +(through the :meth:`~XPathItemParser.add_xpath` or +:meth:`~ItemParser.add_value` methods) and the result of the input parser is +collected and kept inside the ItemParser. After collecting all data, the +:meth:`ItemParser.populate_item` method is called to populate and get the +populated :class:`~scrapy.newitem.Item` object. That's when the output parser +is called with the data previously collected (and processed using the input +parser). The result of the output parser is the final value that gets assigned +to the item. + +Let's see an example to illustrate how this input and output parsers are +called for a particular field (the same applies for any other field):: + + p = XPathItemParser(Product(), some_xpath_selector) + p.add_xpath('name', xpath1) # (1) + p.add_xpath('name', xpath2) # (2) + return p.populate_item() # (3) + +So what happens is: + +1. Data from ``xpath1`` is extracted, and passed through the *input parser* of + the ``name`` field. The result of the input parser is collected and kept in + the Item Parser (but not yet assigned to the item). + +2. Data from ``xpath2`` is extracted, and passed through the same *input + parser* used in (1). The result of the input parser is appended to the data + collected in (1) (if any). + +3. The data collected in (1) and (2) is passed through the *output parser* of + the ``name`` field. The result of the output parser is the value assigned to + the ``name`` field in the item. + +It's worth noticing that parsers are just callable objects, which are called +with the data to be parsed, and return a parsed value. So you can use any +function as input or output parser, provided they can receive only one +positional (required) argument. + +The other thing you need to keep in mind is that the values returned by input +parsers are collected internally (in lists) and then passed to output parsers +to populate the fields, so output parsers should expect iterables as input. + +Last, but not least, Scrapy comes with some :ref:`commonly used parsers +` built-in for convenience. + + +Declaring Item Parsers +====================== + +Item Parsers are declared like Items, by using a class definition syntax. Here +is an example:: + + from scrapy.contrib.itemparser import ItemParser + from scrapy.contrib.itemparser.parsers import TakeFirst, ApplyConcat, Join + + class ProductParser(ItemParser): + + default_expander = TakeFirst() + + name_in = ApplyConcat(unicode.title) + name_out = Join() + + price_in = ApplyConcat(unicode.strip) + price_out = TakeFirst() + + # ... + +As you can see, input parsers are declared using the ``_in`` suffix while +output parsers are declared using the ``_out`` suffix. And you can also declare +a default input/output parsers using the +:attr:`ItemParser.default_input_parser` and +:attr:`ItemParser.default_output_parser` attributes. + +.. _topics-itemparser-parsers-declaring: + +Declaring Input and Output Parsers +================================== + +As seen in the previous section, input and output parsers can be declared in +the Item Parser definition, and it's very common to declare input parsers this +way. However, there is one more place where you can specify the input and +output parsers to use: in the :ref:`Item Field ` +metadata. Here is an example:: + + from scrapy.newitem import Item, Field + from scrapy.contrib.itemparser.parser import ApplyConcat, Join, TakeFirst + + from scrapy.utils.markup import remove_entities + from myproject.utils import filter_prices + + class Product(Item): + name = Field( + input_parser=ApplyConcat(remove_entities), + output_parser=Join(), + ) + price = Field( + default=0, + input_parser=ApplyConcat(remove_entities, filter_prices), + output_parser=TakeFirst(), + ) + +The precedence order, for both input and output parsers, is as follows: + +1. Item Parser field-specific attributes: ``field_in`` and ``field_out`` (most + precedence) +2. Field metadata (``input_parser`` and ``output_parser`` key) +3. Item Parser defaults: :meth:`ItemParser.default_expander` and + :meth:`ItemParser.default_output_parser` (least precedence) + +See also: :ref:`topics-itemparser-extending`. + +.. _topics-itemparser-context: + +Item Parser Context +=================== + +The Item Parser Context is a dict of arbitrary key/values which is shared among +all input and output parsers in the Item Parser. It can be passed when +declaring, instantiating or using Item Parser. They are used to modify the +behaviour of the input/output parsers. + +For example, suppose you have a function ``parse_length`` which receives a text +value and extracts a length from it:: + + def parse_length(text, parser_context): + unit = parser_context.get('unit', 'm') + # ... length parsing code goes here ... + return parsed_length + +By accepting a ``parser_context`` argument the function is explicitly telling +the Item Parser that is able to receive an Item Parser context, so the Item +Parser passes the currently active context when calling it, and the parser +function (``parse_length`` in this case) can thus use them. + +There are several ways to modify Item Parser context values: + +1. By modifying the currently active Item Parser context +(:meth:`ItemParser.context` attribute):: + + parser = ItemParser(product, unit='cm') + parser.context['unit'] = 'cm' + +2. On Item Parser instantiation (the keyword arguments of Item Parser + constructor are stored in the Item Parser context):: + + p = ItemParser(product, unit='cm') + +2. On Item Parser declaration, for those input/output parsers that support + instatiating them with a Item Parser context. :class:`ApplyConcat` is one of + them:: + + class ProductParser(ItemParser): + length_out = ApplyConcat(parse_length, unit='cm') + + +ItemParser objects +================== + +.. class:: ItemParser([item], \**kwargs) + + Return a new Item Parser for populating the given Item. If no item is + given, one is instantiated automatically using the class in + :attr:`default_item_class`. + + The item and the remaining keyword arguments are assigned to the Parser + context (accesible through the :attr:`context` attribute). + + .. method:: add_value(field_name, value) + + Add the given ``value`` for the given field. + + The value is passed through the :ref:`field input parser + ` and its result appened to the data + collected for that field. If the field already contains collected data, + the new data is added. + + Examples:: + + parser.add_value('name', u'Color TV') + parser.add_value('colours', [u'white', u'blue']) + parser.add_value('length', u'100', default_unit='cm') + + .. method:: replace_value(field_name, value) + + Similar to :meth:`add_value` but replaces the collected data with the + new value instead of adding it. + + .. method:: populate_item() + + Populate the item with the data collected so far, and return it. The + data collected is first passed through the :ref:`field output parsers + ` to get the final value to assign to each + item field. + + .. method:: get_collected_values(field_name) + + Return the collected values for the given field. + + .. method:: get_output_value(field_name) + + Return the collected values parsed using the output parser, for the + given field. This method doesn't populate or modify the item at all. + + .. method:: get_input_parser(field_name) + + Return the input parser for the given field. + + .. method:: get_output_parser(field_name) + + Return the output parser for the given field. + + .. attribute:: item + + The :class:`~scrapy.newitem.Item` object being parsed by this Item + Parser. + + .. attribute:: context + + The currently active :ref:`Context ` of this + Item Parser. + + .. attribute:: default_item_class + + An Item class (or factory), used to instantiate items when not given in + the constructor. + + .. attribute:: default_input_parser + + The default input parser to use for those fields which don't specify + one. + + .. attribute:: default_output_parser + + The default output parser to use for those fields which don't specify + one. + +.. class:: XPathItemParser([item, selector, response], \**kwargs) + + The :class:`XPathItemParser` class extends the :class:`ItemParser` class + providing more convenient mechanisms for extracting data from web pages + using :ref:`XPath selectors `. + + :class:`XPathItemParser` objects accept two more additional parameters in + their constructors: + + :param selector: The selector to extract data from, when using the + :meth:`add_xpath` or :meth:`replace_xpath` method. + :type selector: :class:`~scrapy.xpath.XPathSelector` object + + :param response: The response used to construct the selector using the + :attr:`default_selector_class`, unless the selector argument is given, + in which case this argument is ignored. + :type response: :class:`~scrapy.http.Response` object + + .. method:: add_xpath(field_name, xpath, re=None) + + Similar to :meth:`ItemParser.add_value` but receives an XPath instead of a + value, which is used to extract a list of unicode strings from the + selector associated with this :class:`XPathItemParser`. If the ``re`` + argument is given, it's used for extrating data from the selector using + the :meth:`~scrapy.xpath.XPathSelector.re` method. + + :param xpath: the XPath to extract data from + :type xpath: str + + :param re: a regular expression to use for extracting data from the + selected XPath region + :type re: str or compiled regex + + Examples:: + + # HTML snippet:

Color TV

+ parser.add_xpath('name', '//p[@class="product-name"]') + # HTML snippet:

the price is $1200

+ parser.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') + + .. method:: replace_xpath(field_name, xpath, re=None) + + Similar to :meth:`add_xpath` but replaces collected data instead of + adding it. + + .. attribute:: default_selector_class + + The class used to construct the :attr:`selector` of this + :class:`XPathItemParser`, if only a response is given in the constructor. + If a selector is given in the constructor this attribute is ignored. + This attribute is sometimes overridden in subclasses. + + .. attribute:: selector + + The :class:`~scrapy.xpath.XPathSelector` object to extract data from. + It's either the selector given in the constructor or one created from + the response given in the constructor using the + :attr:`default_selector_class`. This attribute is meant to be + read-only. + +.. _topics-itemparser-extending: + +Reusing and extending Item Parsers +================================== + +As your project grows bigger and acquires more and more spiders, maintenance +becomes a fundamental problem, specially when you have to deal with many +different parsing rules for each spider, having a lot of exceptions, but also +wanting to reuse the common parsers. + +Item Parsers are designed to ease the maintenance burden of parsing rules, +without loosing flexibility and, at the same time, providing a convenient +mechanism for extending and overriding them. For this reason Item Parsers +support traditional Python class inheritance for dealing with differences of +specific spiders (or group of spiders). + +Suppose, for example, that some particular site encloses their product names in +three dashes (ie. ``---Plasma TV---``) and you don't want to end up scraping +those dashes in the final product names. + +Here's how you can remove those dashes by reusing and extending the default +Product Item Parser (``ProductParser``):: + + from scrapy.contrib.itemparser.parsers import ApplyConcat + from myproject.itemparsers import ProductParser + + def strip_dashes(x): + return x.strip('-') + + class SiteSpecificParser(ProductParser): + name_in = ApplyConcat(ProductParser.name_in, strip_dashes) + +Another case where extending Item Parsers can be very helpful is when you have +multiple source formats, for example XML and HTML. In the XML version you may +want to remove ``CDATA`` occurrences. Here's an example of how to do it:: + + from scrapy.contrib.itemparser.parsers import ApplyConcat + from myproject.itemparsers import ProductParser + from myproject.utils.xml import remove_cdata + + class XmlProductParser(ProductParser): + name_in = ApplyConcat(remove_cdata, ProductParser.name_in) + +And that's how you typically extend input parsers. + +As for output parsers, it is more common to declare them in the field metadata, +as they usually depend only on the field and not on each specific site parsing +rule (as input parsers do). See also: +:ref:`topics-itemparser-parsers-declaring`. + +There are many other possible ways to extend, inherit and override your Item +Parsers, and different Item Parsers hierarchies may fit better for different +projects. Scrapy only provides the mechanism, it doesn't impose any specific +organization of your Parsers collection - that's up to you and your project +needs. + +.. _topics-itemparser-available-parsers: + +Available built-in parsers +========================== + +Even though you can use any callable function as input and output parsers, +Scrapy provides some commonly used parsers, which are described below. Some of +them, like the :class:`ApplyConcat` (which is typically used as input parser) +composes the output of several functions executed in order, to produce the +final parsed value. + +Here is a list of all built-in parsers: + +.. _topics-itemparser-Applyconcat: + +ApplyConcat parser +------------------ + +The ApplyConcat parser is the recommended parser to use if you want to +concatenate the processing of several functions in a pipeline. + +.. module:: scrapy.contrib.itemparser.parsers + :synopsis: Parser functions to use with Item Parsers + +.. class:: ApplyConcat(\*functions, \**default_parser_context) + + A parser which applies the given functions consecutively, in order, + concatenating their results before next function call. So each function + returns a list of values (though it could return ``None`` or a signle value + too) and the next function is called once for each of those values, + receiving one of those values as input each time. The output of each + function call (for each input value) is concatenated and each values of the + concatenation is used to call the next function, and the process repeats + until there are no functions left. + + Each function can optionally receive a ``parser_context`` parameter, which + will contain the currently active :ref:`Item Parser context + `. + + The keyword arguments passed in the consturctor are used as the default + Item Parser context values passed on each function call. However, the final + Item Parser context values passed to funtions get overriden with the + currently active Item Parser context accesible through the + :meth:`ItemParser.context` attribute. + + Example:: + + >>> def filter_world(x): + ... return None if x == 'world' else x + ... + >>> from scrapy.contrib.itemparser.parsers import ApplyConcat + >>> parser = ApplyConcat(filter_world, str.upper) + >>> parser(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] + +.. class:: TakeFirst + + Return the first non null/empty value from the values to received, so it's + typically used as output parser of single-valued fields. It doesn't receive + any constructor arguments, nor accepts a Item Parser context. + + Example:: + + >>> from scrapy.contrib.itemparser.parsers import TakeFirst + >>> parser = TakeFirst() + >>> parser(['', 'one', 'two', 'three']) + 'one' + +.. class:: Identity + + Return the original values unchanged. It doesn't receive any constructor + arguments nor accepts a Item Parser context. + + Example:: + + >>> from scrapy.contrib.itemparser.parsers import Identity + >>> parser = Identity() + >>> parser(['one', 'two', 'three']) + ['one', 'two', 'three'] + +.. class:: Join(separator=u' ') + + Return the values joined with the separator given in the constructor, which + defaults to ``u' '``. It doesn't accept a Item Parser context. + + When using the default separator, this parser is equivalent to the + function: ``u' '.join`` + + Examples:: + + >>> from scrapy.contrib.itemparser.parsers import Join + >>> parser = Join() + >>> parser(['one', 'two', 'three']) + u'one two three' + >>> parser = Join('
') + >>> parser(['one', 'two', 'three']) + u'one
two
three' diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst deleted file mode 100644 index dfec482d2..000000000 --- a/docs/experimental/loaders.rst +++ /dev/null @@ -1,501 +0,0 @@ -.. _topics-loader: - -============ -Item Loaders -============ - -.. module:: scrapy.newitem.loader - :synopsis: Item Loader class - -Item Loaders (or Loaders, for short) provide a convenient mechanism for -populating scraped :ref:`Items `. Even though Items can be -populated using their own dictionary-like API, the Loaders provide a much more -convenient API for populating them from a scraping process, by automating some -common tasks like parsing the raw extracted data before assigning it. - -In other words, :ref:`Items ` provide the *container* of -scraped data, while Loaders provide the mechanism for *populating* that -container. - -Loaders are designed to provide a flexible, efficient and easy mechanism for -extending and overriding different field parsing rules (either by spider, or by -source format) without becoming a nightmare to maintain - -Using Loaders to populate items -=============================== - -To use a Loader, you must first instantiate it. You can either instantiate it -with an Item object or without one, in which case an Item is automatically -instantiated in the Loader constructor using the Item class specified in the -:attr:`Loader.default_item_class` attribute. - -Then, you start adding values to the Loader, typically collecting them using -:ref:`Selectors `. You can add more than one value to the -same item field, the Loader will know how to "join" those values later using a -Reducer. - -Here is a typical Loader usage in a :ref:`Spider `, using the -:ref:`Product item ` declared in the :ref:`Items -section `:: - - from scrapy.item.loader import XPathLoader - from scrapy.xpath import HtmlXPathSelector - from myproject.items import Product - - def parse(self, response): - l = XPathLoader(item=Product(), response=response) - l.add_xpath('name', '//div[@class="product_name"]') - l.add_xpath('name', '//div[@class="product_title"]') - l.add_xpath('price', '//p[@id="price"]') - l.add_xpath('stock', '//p[@id="stock"]') - l.add_value('last_updated', 'today') # you can also use literal values - return l.get_item() - -By quickly looking at that code we can see the ``name`` field is being -extracted from two different XPath locations in the page: - -1. ``//div[@class="product_name"]`` -2. ``//div[@class="product_title"]`` - -In other words, data is being collected by extracting it from two XPath -locations, using the :meth:`~XPathLoader.add_xpath` method. This is the data -that will be assigned to the ``name`` field later. - -Afterwards, similar calls are used for ``price`` and ``stock`` fields, and -finally the ``last_update`` field is populated directly with a literal value -(``today``) using a different method: :meth:`~Loader.add_value`. - -Finally, when all data is collected, the :meth:`Loader.get_item` method is -called which actually populates and returns the item populated with the data -previously extracted and collected with the :meth:`~XPathLoader.add_xpath` and -:meth:`~Loader.add_value` calls. - -.. _topics-loader-expred: - -Expanders and Reducers -====================== - -A Loader is composed of one expander and one reducer for each (item) field. The -Expander processes the extracted data as soon as it's received (through the -:meth:`~XPathLoader.add_xpath` or :meth:`~Loader.add_value` methods) and the -result of the expander is collected and kept inside the Loader. After -collecting all data, the :meth:`Loader.get_item` method is called to actually -populate and get the :class:`~scrapy.newitem.Item` object. That's when the -Reducers are called with the data previously collected (using the Expanders) -and the output of the Reducers are the actual values that get assigned to the -item. - -Let's see an example to illustrate how Expanders and Reducers are called, for a -particular field (the same applies for any other field):: - - l = XPathLoader(Product(), some_selector) - l.add_xpath('name', xpath1) # (1) - l.add_xpath('name', xpath2) # (2) - return l.get_item() # (3) - -So what happens is: - -1. Data from ``xpath1`` is extracted, and passed through the Expander of the - ``name`` field. The output of the expander is collected and kept in the - loader (but not yet assigned to the item). - -2. Data from ``xpath2`` is extracted, and passed through the same Expander used - in (1). The output of the expander is appended to the data collected in (1) - (if any). - -3. The data collected in (1) and (2) is passed through the Reducer of the - ``name`` field. The output of the Reducer is the value assigned to the - ``name`` field in the item. - -Scrapy comes with one major expander built-in, the :ref:`Tree Expander -`, and :ref:`a couple of commonly used reducers -`. - -Declaring Loaders -================= - -Loaders are declared like Items, by using a class definition syntax. Here is an -example:: - - from scrapy.newitem.loader import Loader - from scrapy.newitem.loader.expanders import TreeExpander - from scrapy.newitem.loader.reducers import Join, TakeFirst - - class ProductLoader(Loader): - - default_expander = TakeFirst() - - name_exp = TreeExpander(unicode.title) - name_red = Join() - - price_exp = TreeExpander(unicode.strip) - price_red = TakeFirst() - - ... - -As you can see, expanders are declared using the ``_exp`` suffix while reducers -are declared using the ``_red`` suffix. And you can also declare a default -expander using the :attr:`Loader.default_expander` attribute. - -.. _topics-loader-expred-declaring: - -Declaring Extenders and Reducers -================================ - -As seen in the previous section, extenders and reducers can be declared in the -Loader definition, and it's very common to declare expanders this way. However, -there is one more place where you can specify the exanders and reducers to use: -in the :ref:`Item Field ` metadata. Here is an example:: - - from scrapy.newitem import Item, Field - from scrapy.newitem.loader.expanders import TreeExpander - from scrapy.newitem.loader.reducers import Join, TakeFirst - - from scrapy.utils.markup import remove_entities - from myprojct.utils import filter_prices - - class Product(Item): - name = Field( - expander=TreeExpander(remove_entities), - reducer=Join(), - ) - price = Field( - default=0, - expander=TreeExpander(remove_entities, filter_prices), - reducer=TakeFirst(), - ) - -The precendece order, for both expander and reducer declarations, is as -follows: - -1. Loader field-specific attributes: ``field_exp`` and ``field_red`` (more - precedence) -2. Field metadata (``expander`` and ``reducer`` key) -3. Loader defaults: :meth:`Loader.default_expander` and - :meth:`Loader.default_reducer` (less precedence) - -See also: :ref:`topics-loader-extending`. - -.. _topics-loader-args: - -Item Loader arguments -===================== - -The Loader arguments is a dict of arbitrary key/values which can be passed when -declaring, instantiating or using Loaders. They are used modify the behaviour -of the expanders. - -For example, suppose you have a function ``parse_length`` which receives a text -value and extracts a length from it:: - - def parse_length(text, loader_args): - unit = loader_args('unit', 'm') - # ... length parsing code goes here ... - return parsed_length - -Since it receives a ``loader_args`` the Expander will pass the currently active -Loader arguments when calling it. - -There are seveal ways to pass Loader arguments: - -1. Passing arguments on Loader declaration:: - - class ProductLoader(Loader): - length_exp = TreeExpander(parse_length, unit='cm') - -2. Passing arguments on Loader instantiation:: - - l = Loader(product, unit='cm') - -3. Passing arguments on Loader usage:: - - l.add_xpath('length', '//div', unit='cm') - -Loader objects -============== - -.. class:: Loader([item], \**loader_args) - - Return a new Item Loader for populating the given Item. If no item is - given, one is instantiated using the class in :attr:`default_item_class`. - - .. method:: add_value(field_name, value, \**new_loader_args) - - Add the given ``value`` for the given field. - - The value is passed through the :ref:`field expander - ` and its output appened to the data collected - for that field. If the field already contains collected data, the new - data is added. - - If any keyword arguments are passed, they're used as :ref:`Loader - arguments ` when calling the expanders. - - Examples:: - - loader.add_value('name', u'Color TV') - loader.add_value('colours', [u'white', u'blue']) - loader.add_value('length', u'100', default_unit='cm') - - .. method:: replace_value(field_name, value, \**new_loader_args) - - Similar to :meth:`add_value` but replaces collected data instead of - adding it. - - - .. method:: get_item() - - Populate the item with the data collected so far, and return it. The - data collected is first passed through the :ref:`field reducers - ` to get the final value to assign to each item - field. - - .. method:: get_expanded_value(field_name) - - Return the expanded data for the given field. In other words, return - the dat collected so far for the given field, without reducing it. - - .. method:: get_reduced_value(field_name) - - Return the reduced value for the given field, without modifying the - item. - - .. method:: get_expander(field_name) - - Return the expander for the given field. - - .. method:: get_reducer(field_name) - - Return the reducer for the given field. - - .. attribute:: default_item_class - - An Item class (or factory), used to instantiate items when not given in - the constructor. - - .. attribute:: default_expander - - The default expander to use for those fields which don't define a - specific expander - - .. attribute:: default_reducer - - The default reducer to use for those fields which don't define a - specific expander - -.. class:: XPathLoader([item, selector, response], \**loader_args) - - The :class:`XPathLoader` class extends the :class:`Loader` class providing - more convenient mechanisms for extracting data from web pages using - :ref:`XPath selectors `. - - :class:`XPathLoader` objects accept two more additional parameters in their - constructors: - - :param selector: The selector to extract data from, when using the - :meth:`add_xpath` or :meth:`replace_xpath` method. - :type selector: :class:`~scrapy.xpath.XPathSelector` object - - :param response: The response used to construct the selector using the - :attr:`default_selector_class`, unless the selector argument is given, - in which case this argument is ignored. - :type response: :class:`~scrapy.http.Response` object - - .. method:: add_xpath(field_name, xpath, re=None, \**new_loader_args) - - Similar to :meth:`Loader.add_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`XPathLoader`. If the ``re`` - argument is given, it's used for extrating data from the selector using - the :meth:`~scrapy.xpath.XPathSelector.re` method. - - :param xpath: the XPath to extract data from - :type xpath: str - - :param re: a regular expression to use for extracting data from the - selected XPath region - :type re: str or compiled regex - - Examples:: - - # HTML snippet:

Color TV

- loader.add_xpath('name', '//p[@class="product-name"]') - # HTML snippet:

the price is $1200

- loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - - .. method:: replace_xpath(field_name, xpath, re=None, \**new_loader_args) - - Similar to :meth:`add_xpath` but replaces collected data instead of - adding it. - - .. attribute:: default_selector_class - - The class used to construct the :attr:`selector` of this - :class:`XPathLoader`, if only a response is given in the constructor. - If a selector is given in the constructor this attribute is ignored. - This attribute is sometimes overridden in subclasses. - - .. attribute:: selector - - The :class:`~scrapy.xpath.XPathSelector` object to extract data from. - It's either the selector given in the constructor or one created from - the response given in the constructor using the - :attr:`default_selector_class`. This attribute is meant to be - read-only. - -.. _topics-loader-extending: - -Reusing and extending Loaders -============================= - -As your project grows bigger and acquires more and more spiders, maintenance -becomes a fundamental problem, specially when you have to deal with many -different parsing rules per spider, a lot of exceptions, but also want to reuse -the common cases. - -Loaders are designed to ease the maintenance of parsing rules, without loosing -flexibility and, at the same time, providing a convenient mechanism for -extending and overriding them. For this reason Loaders support traditional -class inheritance for for dealing with differences of specific spiders (or -group of spiders). - -Suppose, for example, that some particular site encloses their product names -between three dashes (ie. ``---Plasma TV---``) and you don't want to end up -scraping those dashes in the final product names. - -Here's how you can remove those dashes by reusing and extending the default -Product Loader:: - - strip_dashes = lambda x: x.strip('-') - - class SiteSpecificLoader(ProductLoader): - name_exp = TreeExpander(ProductLoader.name_exp, strip_dashes) - -Another case where extending Loaders can be very helpful is when you have -multiple source formats, for example XML and HTML. In the XML version you may -want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - - from myproject.utils.xml import remove_cdata - - class XmlLoader(ProductLoader): - name_exp = TreeExpander(remove_cdata, ProductLoader.name_exp) - -And that's how you typically extend expanders. - -As for reducers, it is more common to declare them in the field metadata, as -they usually depend only on the field and not on each specific site parsing -rule (as expanders do). See also: :ref:`topics-loader-expred-declaring`. - -There are many other possible ways to extend, inherit and override your -Loaders, and different Loader hierarchies may fit better for different -projects. Scrapy only provides the mechanism, it doesn't impose any specific -organization of your Loaders collection - that's up to you and your project -needs. - -Available Expanders -=================== - -.. _topics-loader-tree-expander: - -Tree Expander -------------- - -The Tree Expander is the recommended Expander to use and the only really useful -one, as the other is just an identity expander. - -.. module:: scrapy.newitem.loader.expanders - :synopsis: Expander classes to use with Item Loaders - -.. class:: TreeExpander(\*functions, \**default_loader_arguments) - - An expander which applies the given functions consecutively, in order, to - each value returned by the previous function. - - The algorithm consists in an ordered list of functions, each of which - receives one value and can return zero, one or more values (as a list or - iterable). If a function returns more than one value, the next function in - the list will be called with each of those values, potentially returning - more values and thus expanding the execution into different branches, which - is why this expander is called Tree Expander. - - Each expander function can optionally receive a ``loader_args`` argument, - which will contain the currently active :ref:`Loader arguments - `. - - The keyword arguments passed in the consturctor are used as the default - Loader arguments passed to on each expander call. This arguments can be - overriden with specific noader arguments passed on each expander call. - - Example:: - - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.newitem.loader.expanders import TreeExpander - >>> expander = TreeExpander(filter_world, str.upper) - >>> expander(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] - - -IdentityExpander ----------------- - -.. class:: IdentityExpander - - An expander which returns the original values unchanged. It doesn't support - any constructor arguments. - -.. _topics-loader-reducers: - -Available Reducers -================== - -.. module:: scrapy.newitem.loader.reducers - :synopsis: Reducer classes to use with Item Loaders - -Reducers are callable objects which are called with a list of values (to be -reduced) as their first and only argument. Scrapy provides some simple, -commonly used reducers, which are described below. But you can use any function -or callable as reducer. - -.. class:: TakeFirst - - Return the first non-null value from the values to reduce, so it's used for - single-valued fields. It doesn't receive any constructor arguments. - - Example:: - - >>> from scrapy.newitem.loader.reducers import TakeFirst - >>> reducer = TakeFirst() - >>> reducer(['', 'one', 'two', 'three']) - 'one' - -.. class:: Identity - - Return the values to reduce unchanged, so it's used for multi-valued - fields. It doesn't receive any constructor arguments. - - Example:: - - >>> from scrapy.newitem.loader.reducers import Identity - >>> reducer = Identity() - >>> reducer(['one', 'two', 'three']) - ['one', 'two', 'three'] - -.. class:: Join(separator=u' ') - - Return a the values to reduce joined with the separator given in the - constructor, which defaults to ``u' '``. - - When using the default separator, this reducer is equivalent to the - function: ``u' '.join`` - - Examples:: - - >>> from scrapy.newitem.loader.reducers import Join - >>> reducer = Join() - >>> reducer(['one', 'two', 'three']) - u'one two three' - >>> reducer = Join('
') - >>> reducer(['one', 'two', 'three']) - u'one
two
three' diff --git a/scrapy/contrib/itemparser/__init__.py b/scrapy/contrib/itemparser/__init__.py new file mode 100644 index 000000000..250453592 --- /dev/null +++ b/scrapy/contrib/itemparser/__init__.py @@ -0,0 +1,93 @@ +""" +Item Parser + +See documentation in docs/topics/itemparser.rst +""" + +from collections import defaultdict + +from scrapy.newitem import Item +from scrapy.xpath import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter +from .common import wrap_parser_context +from .parsers import Identity + +class ItemParser(object): + + default_item_class = Item + default_input_parser = Identity() + default_output_parser = Identity() + + def __init__(self, item=None, **context): + if item is None: + item = self.default_item_class() + self.item = context['item'] = item + self.context = context + self._values = defaultdict(list) + + def add_value(self, field_name, value): + parsed_value = self._parse_input_value(field_name, value) + self._values[field_name] += arg_to_iter(parsed_value) + + def replace_value(self, field_name, value): + parsed_value = self._parse_input_value(field_name, value) + self._values[field_name] = arg_to_iter(parsed_value) + + def populate_item(self): + item = self.item + for field_name in self._values: + item[field_name] = self.get_output_value(field_name) + return item + + def get_output_value(self, field_name): + parser = self.get_output_parser(field_name) + parser = wrap_parser_context(parser, self.context) + return parser(self._values[field_name]) + + def get_collected_values(self, field_name): + return self._values[field_name] + + def get_input_parser(self, field_name): + parser = getattr(self, '%s_in' % field_name, None) + if not parser: + parser = self.item.fields[field_name].get('input_parser', \ + self.default_input_parser) + return parser + + def get_output_parser(self, field_name): + parser = getattr(self, '%s_out' % field_name, None) + if not parser: + parser = self.item.fields[field_name].get('output_parser', \ + self.default_output_parser) + return parser + + def _parse_input_value(self, field_name, value): + parser = self.get_input_parser(field_name) + parser = wrap_parser_context(parser, self.context) + return parser(value) + + +class XPathItemParser(ItemParser): + + default_selector_class = HtmlXPathSelector + + def __init__(self, item=None, selector=None, response=None, **context): + if selector is None and response is None: + raise RuntimeError("%s must be instantiated with a selector" \ + "or response" % self.__class__.__name__) + if selector is None: + selector = self.default_selector_class(response) + self.selector = selector + context.update(selector=selector, response=response) + super(XPathItemParser, self).__init__(item, **context) + + def add_xpath(self, field_name, xpath, re=None): + self.add_value(field_name, self._get_values(field_name, xpath, re)) + + def replace_xpath(self, field_name, xpath, re=None): + self.replace_value(field_name, self._get_values(field_name, xpath, re)) + + def _get_values(self, field_name, xpath, re): + x = self.selector.x(xpath) + return x.re(re) if re else x.extract() + diff --git a/scrapy/contrib/itemparser/common.py b/scrapy/contrib/itemparser/common.py new file mode 100644 index 000000000..a1a8ac069 --- /dev/null +++ b/scrapy/contrib/itemparser/common.py @@ -0,0 +1,13 @@ +"""Common functions used in Item Parsers code""" + +from functools import partial +from scrapy.utils.python import get_func_args + +def wrap_parser_context(function, context): + """Wrap functions that receive parser_context to contain those parser + arguments pre-loaded and expose a interface that receives only one argument + """ + if 'parser_context' in get_func_args(function): + return partial(function, parser_context=context) + else: + return function diff --git a/scrapy/contrib/itemparser/parsers.py b/scrapy/contrib/itemparser/parsers.py new file mode 100644 index 000000000..f66ce182a --- /dev/null +++ b/scrapy/contrib/itemparser/parsers.py @@ -0,0 +1,52 @@ +""" +This module provides some commonly used parser functions for Item Parsers. + +See documentation in docs/topics/itemparser.rst +""" + +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.datatypes import MergeDict +from .common import wrap_parser_context + +class ApplyConcat(object): + + def __init__(self, *functions, **default_parser_context): + self.functions = functions + self.default_parser_context = default_parser_context + + def __call__(self, value, parser_context=None): + values = arg_to_iter(value) + if parser_context: + context = MergeDict(parser_context, self.default_parser_context) + else: + context = self.default_parser_context + wrapped_funcs = [wrap_parser_context(f, context) for f in self.functions] + for func in wrapped_funcs: + next_values = [] + for v in values: + next_values += arg_to_iter(func(v)) + values = next_values + return list(values) + + +class TakeFirst(object): + + def __call__(self, values): + for value in values: + if value: + return value + + +class Identity(object): + + def __call__(self, values): + return values + + +class Join(object): + + def __init__(self, separator=u' '): + self.separator = separator + + def __call__(self, values): + return self.separator.join(values) diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py deleted file mode 100644 index f40e4c7e5..000000000 --- a/scrapy/newitem/loader/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -from collections import defaultdict - -from scrapy.utils.datatypes import MergeDict -from scrapy.newitem import Item -from scrapy.newitem.loader.reducers import TakeFirst -from scrapy.newitem.loader.expanders import IdentityExpander -from scrapy.xpath import HtmlXPathSelector - -class Loader(object): - - default_item_class = Item - default_expander = IdentityExpander() - default_reducer = TakeFirst() - - def __init__(self, item=None, **loader_args): - if item is None: - item = self.default_item_class() - self._item = loader_args['item'] = item - self._loader_args = loader_args - self._values = defaultdict(list) - - def add_value(self, field_name, value, **new_loader_args): - self._values[field_name] += self._expand_value(field_name, value, \ - new_loader_args) - - def replace_value(self, field_name, value, **new_loader_args): - self._values[field_name] = self._expand_value(field_name, value, \ - new_loader_args) - - def get_item(self): - item = self._item - for field_name in self._values: - item[field_name] = self.get_reduced_value(field_name) - return item - - def get_expanded_value(self, field_name): - return self._values[field_name] - - def get_reduced_value(self, field_name): - reducer = self.get_reducer(field_name) - return reducer(self._values[field_name]) - - def get_expander(self, field_name): - expander = getattr(self, '%s_exp' % field_name, None) - if not expander: - expander = self._item.fields[field_name].get('expander', \ - self.default_expander) - return expander - - def get_reducer(self, field_name): - reducer = getattr(self, '%s_red' % field_name, None) - if not reducer: - reducer = self._item.fields[field_name].get('reducer', \ - self.default_reducer) - return reducer - - def _expand_value(self, field_name, value, new_loader_args): - loader_args = self._loader_args - if new_loader_args: - loader_args = MergeDict(new_loader_args, self._loader_args) - expander = self.get_expander(field_name) - return expander(value, loader_args=loader_args) - - -class XPathLoader(Loader): - - default_selector_class = HtmlXPathSelector - - def __init__(self, item=None, selector=None, response=None, **loader_args): - if selector is None and response is None: - raise RuntimeError("%s must be instantiated with a selector" \ - "or response" % self.__class__.__name__) - if selector is None: - selector = self.default_selector_class(response) - self.selector = selector - loader_args.update(selector=selector, response=response) - super(XPathLoader, self).__init__(item, **loader_args) - - def add_xpath(self, field_name, xpath, re=None, **new_loader_args): - self.add_value(field_name, self._get_values(field_name, xpath, re), - **new_loader_args) - - def replace_xpath(self, field_name, xpath, re=None, **new_loader_args): - self.replace_value(field_name, self._get_values(field_name, xpath, re), \ - **new_loader_args) - - def _get_values(self, field_name, xpath, re): - x = self.selector.x(xpath) - return x.re(re) if re else x.extract() diff --git a/scrapy/newitem/loader/expanders.py b/scrapy/newitem/loader/expanders.py deleted file mode 100644 index 3e7d77e73..000000000 --- a/scrapy/newitem/loader/expanders.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This module provides some commonly used Expanders. - -See documentation in docs/topics/newitem-loader.rst -""" - -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args -from scrapy.utils.datatypes import MergeDict - -class TreeExpander(object): - - def __init__(self, *functions, **default_loader_args): - self.default_loader_args = default_loader_args - self.wrapped_funcs = [] - for func in functions: - if 'loader_args' in get_func_args(func): - wfunc = self.wrap_with_args(func) - else: - wfunc = self.wrap_no_args(func) - self.wrapped_funcs.append(wfunc) - - def wrap_no_args(self, f): - return lambda x, y: f(x) - - def wrap_with_args(self, f): - return lambda x, y: f(x, loader_args=y) - - def __call__(self, value, loader_args=None): - values = arg_to_iter(value) - largs = self.default_loader_args - if loader_args: - largs = MergeDict(loader_args, self.default_loader_args) - for func in self.wrapped_funcs: - next_values = [] - for v in values: - next_values += arg_to_iter(func(v, largs)) - values = next_values - return list(values) - - -class IdentityExpander(object): - - def __call__(self, values, loader_args): - return arg_to_iter(values) diff --git a/scrapy/newitem/loader/reducers.py b/scrapy/newitem/loader/reducers.py deleted file mode 100644 index 8f9a3cc9b..000000000 --- a/scrapy/newitem/loader/reducers.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -This module provides some commonly used Reducers. - -See documentation in docs/topics/newitem-loader.rst -""" - -class TakeFirst(object): - - def __call__(self, values): - for value in values: - if value: - return value - - -class Identity(object): - - def __call__(self, values): - return values - - -class Join(object): - - def __init__(self, separator=u' '): - self.separator = separator - - def __call__(self, values): - return self.separator.join(values) diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py deleted file mode 100644 index a32d2b036..000000000 --- a/scrapy/tests/test_itemloader.py +++ /dev/null @@ -1,272 +0,0 @@ -import unittest - -from scrapy.newitem.loader import Loader, XPathLoader -from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander -from scrapy.newitem.loader.reducers import Join, Identity -from scrapy.newitem import Item, Field -from scrapy.xpath import HtmlXPathSelector -from scrapy.http import HtmlResponse - -# test items - -class NameItem(Item): - name = Field() - -class TestItem(NameItem): - url = Field() - summary = Field() - -# test loaders - -class NameLoader(Loader): - default_item_class = TestItem - -class TestLoader(NameLoader): - name_exp = TreeExpander(lambda v: v.title()) - -class DefaultedLoader(NameLoader): - default_expander = TreeExpander(lambda v: v[:-1]) - -# test expanders - -def expander_func_with_args(value, other=None, loader_args=None): - if 'key' in loader_args: - return loader_args['key'] - return value - -class LoaderTest(unittest.TestCase): - - def test_get_item_using_default_loader(self): - i = TestItem() - i['summary'] = u'lala' - il = Loader(item=i) - il.add_value('name', u'marta') - item = il.get_item() - assert item is i - self.assertEqual(item['summary'], u'lala') - self.assertEqual(item['name'], u'marta') - - def test_get_item_using_custom_loader(self): - il = TestLoader() - il.add_value('name', u'marta') - item = il.get_item() - self.assertEqual(item['name'], u'Marta') - - def test_add_value(self): - il = TestLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_expanded_value('name'), [u'Marta']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - il.add_value('name', u'pepe') - self.assertEqual(il.get_expanded_value('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - def test_replace_value(self): - il = TestLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_expanded_value('name'), [u'Marta']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - il.replace_value('name', u'pepe') - self.assertEqual(il.get_expanded_value('name'), [u'Pepe']) - self.assertEqual(il.get_reduced_value('name'), u'Pepe') - - def test_tree_expander_filter(self): - def filter_world(x): - return None if x == 'world' else x - - expander = TreeExpander(filter_world, str.upper) - self.assertEqual(expander(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) - - def test_tree_expander_multiple_functions(self): - class TestLoader(NameLoader): - name_exp = TreeExpander(lambda v: v.title(), lambda v: v[:-1]) - - il = TestLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Mart') - item = il.get_item() - self.assertEqual(item['name'], u'Mart') - - def test_default_expander(self): - il = DefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mart') - - def test_inherited_default_expander(self): - class InheritDefaultedLoader(DefaultedLoader): - pass - - il = InheritDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mart') - - def test_expander_inheritance(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(lambda v: v.lower()) - - il = ChildLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org') - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - class ChildChildLoader(ChildLoader): - url_exp = TreeExpander(lambda v: v.upper()) - summary_exp = TreeExpander(lambda v: v) - - il = ChildChildLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG') - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - def test_empty_tree_expander(self): - class IdentityDefaultedLoader(DefaultedLoader): - name_exp = TreeExpander() - - il = IdentityDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'marta') - - def test_identity_expander(self): - class IdentityDefaultedLoader(DefaultedLoader): - name_exp = IdentityExpander() - - il = IdentityDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'marta') - - def test_extend_custom_expanders(self): - class ChildLoader(TestLoader): - name_exp = TreeExpander(TestLoader.name_exp, unicode.swapcase) - - il = ChildLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mARTA') - - def test_extend_default_expanders(self): - class ChildDefaultedLoader(DefaultedLoader): - name_exp = TreeExpander(DefaultedLoader.default_expander, unicode.swapcase) - - il = ChildDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'MART') - - def test_reducer_using_function(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class TakeFirstLoader(TestLoader): - name_red = u" ".join - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - - def test_reducer_using_classes(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class TakeFirstLoader(TestLoader): - name_red = Join() - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - - class TakeFirstLoader(TestLoader): - name_red = Join("
") - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar
Ta') - - def test_default_reducer(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class LalaLoader(TestLoader): - default_reducer = Identity() - - il = LalaLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), [u'Mar', u'Ta']) - - def test_expander_args_on_declaration(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args, key=u'val') - - il = ChildLoader() - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_expander_args_on_instantiation(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args) - - il = ChildLoader(key=u'val') - il.add_value('url', u'text') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_expander_args_on_assign(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args) - - il = ChildLoader() - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_item_passed_to_expander_functions(self): - def exp_func(value, loader_args): - return loader_args['item']['name'] - - class ChildLoader(TestLoader): - url_exp = TreeExpander(exp_func) - - it = TestItem(name='marta') - il = ChildLoader(item=it) - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'marta') - - def test_add_value_on_unknown_field(self): - il = TestLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) - - -class TestXPathLoader(XPathLoader): - default_item_class = TestItem - name_exp = TreeExpander(lambda v: v.title()) - -class XPathLoaderTest(unittest.TestCase): - - def test_constructor_errors(self): - self.assertRaises(RuntimeError, XPathLoader) - - def test_constructor_with_selector(self): - sel = HtmlXPathSelector(text=u"
marta
") - l = TestXPathLoader(selector=sel) - self.assert_(l.selector is sel) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_reduced_value('name'), u'Marta') - - def test_constructor_with_response(self): - response = HtmlResponse(url="", body="
marta
") - l = TestXPathLoader(response=response) - self.assert_(l.selector) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_reduced_value('name'), u'Marta') - - def test_add_xpath_re(self): - response = HtmlResponse(url="", body="
marta
") - l = TestXPathLoader(response=response) - l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_reduced_value('name'), u'Ma') - - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/tests/test_itemparser.py b/scrapy/tests/test_itemparser.py new file mode 100644 index 000000000..cf0e8addc --- /dev/null +++ b/scrapy/tests/test_itemparser.py @@ -0,0 +1,280 @@ +import unittest + +from scrapy.contrib.itemparser import ItemParser, XPathItemParser +from scrapy.contrib.itemparser.parsers import ApplyConcat, Join, Identity +from scrapy.newitem import Item, Field +from scrapy.xpath import HtmlXPathSelector +from scrapy.http import HtmlResponse + +# test items + +class NameItem(Item): + name = Field() + +class TestItem(NameItem): + url = Field() + summary = Field() + +# test item parsers + +class NameItemParser(ItemParser): + default_item_class = TestItem + +class TestItemParser(NameItemParser): + name_in = ApplyConcat(lambda v: v.title()) + +class DefaultedItemParser(NameItemParser): + default_input_parser = ApplyConcat(lambda v: v[:-1]) + +# test parsers + +def parser_with_args(value, other=None, parser_context=None): + if 'key' in parser_context: + return parser_context['key'] + return value + +class ItemParserTest(unittest.TestCase): + + def test_populate_item_using_default_loader(self): + i = TestItem() + i['summary'] = u'lala' + ip = ItemParser(item=i) + ip.add_value('name', u'marta') + item = ip.populate_item() + assert item is i + self.assertEqual(item['summary'], u'lala') + self.assertEqual(item['name'], [u'marta']) + + def test_populate_item_using_custom_loader(self): + ip = TestItemParser() + ip.add_value('name', u'marta') + item = ip.populate_item() + self.assertEqual(item['name'], [u'Marta']) + + def test_add_value(self): + ip = TestItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_collected_values('name'), [u'Marta']) + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + ip.add_value('name', u'pepe') + self.assertEqual(ip.get_collected_values('name'), [u'Marta', u'Pepe']) + self.assertEqual(ip.get_output_value('name'), [u'Marta', u'Pepe']) + + def test_replace_value(self): + ip = TestItemParser() + ip.replace_value('name', u'marta') + self.assertEqual(ip.get_collected_values('name'), [u'Marta']) + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + ip.replace_value('name', u'pepe') + self.assertEqual(ip.get_collected_values('name'), [u'Pepe']) + self.assertEqual(ip.get_output_value('name'), [u'Pepe']) + + def test_map_concat_filter(self): + def filter_world(x): + return None if x == 'world' else x + + parser = ApplyConcat(filter_world, str.upper) + self.assertEqual(parser(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) + + def test_map_concat_filter_multiple_functions(self): + class TestItemParser(NameItemParser): + name_in = ApplyConcat(lambda v: v.title(), lambda v: v[:-1]) + + ip = TestItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Mart']) + item = ip.populate_item() + self.assertEqual(item['name'], [u'Mart']) + + def test_default_input_parser(self): + ip = DefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mart']) + + def test_inherited_default_input_parser(self): + class InheritDefaultedItemParser(DefaultedItemParser): + pass + + ip = InheritDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mart']) + + def test_input_parser_inheritance(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(lambda v: v.lower()) + + ip = ChildItemParser() + ip.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(ip.get_output_value('url'), [u'http://scrapy.org']) + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + + class ChildChildItemParser(ChildItemParser): + url_in = ApplyConcat(lambda v: v.upper()) + summary_in = ApplyConcat(lambda v: v) + + ip = ChildChildItemParser() + ip.add_value('url', u'http://scrapy.org') + self.assertEqual(ip.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + + def test_empty_map_concat(self): + class IdentityDefaultedItemParser(DefaultedItemParser): + name_in = ApplyConcat() + + ip = IdentityDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'marta']) + + def test_identity_input_parser(self): + class IdentityDefaultedItemParser(DefaultedItemParser): + name_in = Identity() + + ip = IdentityDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'marta']) + + def test_extend_custom_input_parsers(self): + class ChildItemParser(TestItemParser): + name_in = ApplyConcat(TestItemParser.name_in, unicode.swapcase) + + ip = ChildItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mARTA']) + + def test_extend_default_input_parsers(self): + class ChildDefaultedItemParser(DefaultedItemParser): + name_in = ApplyConcat(DefaultedItemParser.default_input_parser, unicode.swapcase) + + ip = ChildDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'MART']) + + def test_output_parser_using_function(self): + ip = TestItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemParser(TestItemParser): + name_out = u" ".join + + ip = TakeFirstItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar Ta') + + def test_output_parser_using_classes(self): + ip = TestItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemParser(TestItemParser): + name_out = Join() + + ip = TakeFirstItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar Ta') + + class TakeFirstItemParser(TestItemParser): + name_out = Join("
") + + ip = TakeFirstItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar
Ta') + + def test_default_output_parser(self): + ip = TestItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class LalaItemParser(TestItemParser): + default_output_parser = Identity() + + ip = LalaItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + def test_parser_context_on_declaration(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser_with_args, key=u'val') + + ip = ChildItemParser() + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_parser_context_on_instantiation(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser_with_args) + + ip = ChildItemParser(key=u'val') + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_parser_context_on_assign(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser_with_args) + + ip = ChildItemParser() + ip.context['key'] = u'val' + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_item_passed_to_input_parser_functions(self): + def parser(value, parser_context): + return parser_context['item']['name'] + + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser) + + it = TestItem(name='marta') + ip = ChildItemParser(item=it) + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['marta']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['marta']) + + def test_add_value_on_unknown_field(self): + ip = TestItemParser() + self.assertRaises(KeyError, ip.add_value, 'wrong_field', [u'lala', u'lolo']) + + +class TestXPathItemParser(XPathItemParser): + default_item_class = TestItem + name_in = ApplyConcat(lambda v: v.title()) + +class XPathItemParserTest(unittest.TestCase): + + def test_constructor_errors(self): + self.assertRaises(RuntimeError, XPathItemParser) + + def test_constructor_with_selector(self): + sel = HtmlXPathSelector(text=u"
marta
") + l = TestXPathItemParser(selector=sel) + self.assert_(l.selector is sel) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'Marta']) + + def test_constructor_with_response(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathItemParser(response=response) + self.assert_(l.selector) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'Marta']) + + def test_add_xpath_re(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathItemParser(response=response) + l.add_xpath('name', '//div/text()', re='ma') + self.assertEqual(l.get_output_value('name'), [u'Ma']) + + +if __name__ == "__main__": + unittest.main() + From 1dc592882b3eb62b3c95140e00da7a81ceaec8c0 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 12 Aug 2009 16:49:07 -0300 Subject: [PATCH 3/4] Moved Item Loader to its final location in scrapy.contrib.loader, and updated doc/tests --HG-- rename : docs/experimental/itemparser.rst => docs/experimental/loaders.rst rename : scrapy/contrib/itemparser/__init__.py => scrapy/contrib/loader/__init__.py rename : scrapy/contrib/itemparser/common.py => scrapy/contrib/loader/common.py rename : scrapy/contrib/itemparser/parsers.py => scrapy/contrib/loader/processor.py rename : scrapy/tests/test_itemparser.py => scrapy/tests/test_contrib_loader.py --- docs/experimental/itemparser.rst | 526 ----------------- docs/experimental/loaders.rst | 527 ++++++++++++++++++ scrapy/contrib/itemparser/common.py | 13 - .../{itemparser => loader}/__init__.py | 54 +- scrapy/contrib/loader/common.py | 13 + .../parsers.py => loader/processor.py} | 20 +- ...t_itemparser.py => test_contrib_loader.py} | 168 +++--- 7 files changed, 661 insertions(+), 660 deletions(-) delete mode 100644 docs/experimental/itemparser.rst create mode 100644 docs/experimental/loaders.rst delete mode 100644 scrapy/contrib/itemparser/common.py rename scrapy/contrib/{itemparser => loader}/__init__.py (63%) create mode 100644 scrapy/contrib/loader/common.py rename scrapy/contrib/{itemparser/parsers.py => loader/processor.py} (60%) rename scrapy/tests/{test_itemparser.py => test_contrib_loader.py} (62%) diff --git a/docs/experimental/itemparser.rst b/docs/experimental/itemparser.rst deleted file mode 100644 index 160ebe698..000000000 --- a/docs/experimental/itemparser.rst +++ /dev/null @@ -1,526 +0,0 @@ -.. _topics-itemparser: - -============ -Item Parsers -============ - -.. module:: scrapy.contrib.itemparser - :synopsis: Item Parser class - -Item Parser provide a convenient mechanism for populating scraped :ref:`Items -`. Even though Items can be populated using their own -dictionary-like API, the Item Parsers provide a much more convenient API for -populating them from a scraping process, by automating some common tasks like -parsing the raw extracted data before assigning it. - -In other words, :ref:`Items ` provide the *container* of -scraped data, while Item Parsers provide the mechanism for *populating* that -container. - -Item Parsers are designed to provide a flexible, efficient and easy mechanism -for extending and overriding different field parsing rules, either by spider, -or by source format (HTML, XML, etc) without becoming a nightmare to maintain. - -Using Item Parsers to populate items -==================================== - -To use an Item Parser, you must first instantiate it. You can either -instantiate it with an Item object or without one, in which case an Item is -automatically instantiated in the Item Parser constructor using the Item class -specified in the :attr:`ItemParser.default_item_class` attribute. - -Then, you start collecting values into the Item Parser, typically using using -:ref:`XPath Selectors `. You can add more than one value to -the same item field, the Item Parser will know how to "join" those values later -using a proper parser function. - -Here is a typical Item Parser usage in a :ref:`Spider `, using -the :ref:`Product item ` declared in the :ref:`Items -chapter `:: - - from scrapy.contrib.itemparser import XPathItemParser - from scrapy.xpath import HtmlXPathSelector - from myproject.items import Product - - def parse(self, response): - p = XPathItemParser(item=Product(), response=response) - p.add_xpath('name', '//div[@class="product_name"]') - p.add_xpath('name', '//div[@class="product_title"]') - p.add_xpath('price', '//p[@id="price"]') - p.add_xpath('stock', '//p[@id="stock"]') - p.add_value('last_updated', 'today') # you can also use literal values - return p.populate_item() - -By quickly looking at that code we can see the ``name`` field is being -extracted from two different XPath locations in the page: - -1. ``//div[@class="product_name"]`` -2. ``//div[@class="product_title"]`` - -In other words, data is being collected by extracting it from two XPath -locations, using the :meth:`~XPathItemParser.add_xpath` method. This is the data -that will be assigned to the ``name`` field later. - -Afterwards, similar calls are used for ``price`` and ``stock`` fields, and -finally the ``last_update`` field is populated directly with a literal value -(``today``) using a different method: :meth:`~ItemParser.add_value`. - -Finally, when all data is collected, the :meth:`ItemParser.populate_item` -method is called which actually populates and returns the item populated with -the data previously extracted and collected with the -:meth:`~XPathItemParser.add_xpath` and :meth:`~ItemParser.add_value` calls. - -.. _topics-itemparser-parsers: - -Input and Output parsers -======================== - -An Item Parser contains one input parser and one output parser for each (item) -field. The input parser processes the extracted data as soon as it's received -(through the :meth:`~XPathItemParser.add_xpath` or -:meth:`~ItemParser.add_value` methods) and the result of the input parser is -collected and kept inside the ItemParser. After collecting all data, the -:meth:`ItemParser.populate_item` method is called to populate and get the -populated :class:`~scrapy.newitem.Item` object. That's when the output parser -is called with the data previously collected (and processed using the input -parser). The result of the output parser is the final value that gets assigned -to the item. - -Let's see an example to illustrate how this input and output parsers are -called for a particular field (the same applies for any other field):: - - p = XPathItemParser(Product(), some_xpath_selector) - p.add_xpath('name', xpath1) # (1) - p.add_xpath('name', xpath2) # (2) - return p.populate_item() # (3) - -So what happens is: - -1. Data from ``xpath1`` is extracted, and passed through the *input parser* of - the ``name`` field. The result of the input parser is collected and kept in - the Item Parser (but not yet assigned to the item). - -2. Data from ``xpath2`` is extracted, and passed through the same *input - parser* used in (1). The result of the input parser is appended to the data - collected in (1) (if any). - -3. The data collected in (1) and (2) is passed through the *output parser* of - the ``name`` field. The result of the output parser is the value assigned to - the ``name`` field in the item. - -It's worth noticing that parsers are just callable objects, which are called -with the data to be parsed, and return a parsed value. So you can use any -function as input or output parser, provided they can receive only one -positional (required) argument. - -The other thing you need to keep in mind is that the values returned by input -parsers are collected internally (in lists) and then passed to output parsers -to populate the fields, so output parsers should expect iterables as input. - -Last, but not least, Scrapy comes with some :ref:`commonly used parsers -` built-in for convenience. - - -Declaring Item Parsers -====================== - -Item Parsers are declared like Items, by using a class definition syntax. Here -is an example:: - - from scrapy.contrib.itemparser import ItemParser - from scrapy.contrib.itemparser.parsers import TakeFirst, ApplyConcat, Join - - class ProductParser(ItemParser): - - default_expander = TakeFirst() - - name_in = ApplyConcat(unicode.title) - name_out = Join() - - price_in = ApplyConcat(unicode.strip) - price_out = TakeFirst() - - # ... - -As you can see, input parsers are declared using the ``_in`` suffix while -output parsers are declared using the ``_out`` suffix. And you can also declare -a default input/output parsers using the -:attr:`ItemParser.default_input_parser` and -:attr:`ItemParser.default_output_parser` attributes. - -.. _topics-itemparser-parsers-declaring: - -Declaring Input and Output Parsers -================================== - -As seen in the previous section, input and output parsers can be declared in -the Item Parser definition, and it's very common to declare input parsers this -way. However, there is one more place where you can specify the input and -output parsers to use: in the :ref:`Item Field ` -metadata. Here is an example:: - - from scrapy.newitem import Item, Field - from scrapy.contrib.itemparser.parser import ApplyConcat, Join, TakeFirst - - from scrapy.utils.markup import remove_entities - from myproject.utils import filter_prices - - class Product(Item): - name = Field( - input_parser=ApplyConcat(remove_entities), - output_parser=Join(), - ) - price = Field( - default=0, - input_parser=ApplyConcat(remove_entities, filter_prices), - output_parser=TakeFirst(), - ) - -The precedence order, for both input and output parsers, is as follows: - -1. Item Parser field-specific attributes: ``field_in`` and ``field_out`` (most - precedence) -2. Field metadata (``input_parser`` and ``output_parser`` key) -3. Item Parser defaults: :meth:`ItemParser.default_expander` and - :meth:`ItemParser.default_output_parser` (least precedence) - -See also: :ref:`topics-itemparser-extending`. - -.. _topics-itemparser-context: - -Item Parser Context -=================== - -The Item Parser Context is a dict of arbitrary key/values which is shared among -all input and output parsers in the Item Parser. It can be passed when -declaring, instantiating or using Item Parser. They are used to modify the -behaviour of the input/output parsers. - -For example, suppose you have a function ``parse_length`` which receives a text -value and extracts a length from it:: - - def parse_length(text, parser_context): - unit = parser_context.get('unit', 'm') - # ... length parsing code goes here ... - return parsed_length - -By accepting a ``parser_context`` argument the function is explicitly telling -the Item Parser that is able to receive an Item Parser context, so the Item -Parser passes the currently active context when calling it, and the parser -function (``parse_length`` in this case) can thus use them. - -There are several ways to modify Item Parser context values: - -1. By modifying the currently active Item Parser context -(:meth:`ItemParser.context` attribute):: - - parser = ItemParser(product, unit='cm') - parser.context['unit'] = 'cm' - -2. On Item Parser instantiation (the keyword arguments of Item Parser - constructor are stored in the Item Parser context):: - - p = ItemParser(product, unit='cm') - -2. On Item Parser declaration, for those input/output parsers that support - instatiating them with a Item Parser context. :class:`ApplyConcat` is one of - them:: - - class ProductParser(ItemParser): - length_out = ApplyConcat(parse_length, unit='cm') - - -ItemParser objects -================== - -.. class:: ItemParser([item], \**kwargs) - - Return a new Item Parser for populating the given Item. If no item is - given, one is instantiated automatically using the class in - :attr:`default_item_class`. - - The item and the remaining keyword arguments are assigned to the Parser - context (accesible through the :attr:`context` attribute). - - .. method:: add_value(field_name, value) - - Add the given ``value`` for the given field. - - The value is passed through the :ref:`field input parser - ` and its result appened to the data - collected for that field. If the field already contains collected data, - the new data is added. - - Examples:: - - parser.add_value('name', u'Color TV') - parser.add_value('colours', [u'white', u'blue']) - parser.add_value('length', u'100', default_unit='cm') - - .. method:: replace_value(field_name, value) - - Similar to :meth:`add_value` but replaces the collected data with the - new value instead of adding it. - - .. method:: populate_item() - - Populate the item with the data collected so far, and return it. The - data collected is first passed through the :ref:`field output parsers - ` to get the final value to assign to each - item field. - - .. method:: get_collected_values(field_name) - - Return the collected values for the given field. - - .. method:: get_output_value(field_name) - - Return the collected values parsed using the output parser, for the - given field. This method doesn't populate or modify the item at all. - - .. method:: get_input_parser(field_name) - - Return the input parser for the given field. - - .. method:: get_output_parser(field_name) - - Return the output parser for the given field. - - .. attribute:: item - - The :class:`~scrapy.newitem.Item` object being parsed by this Item - Parser. - - .. attribute:: context - - The currently active :ref:`Context ` of this - Item Parser. - - .. attribute:: default_item_class - - An Item class (or factory), used to instantiate items when not given in - the constructor. - - .. attribute:: default_input_parser - - The default input parser to use for those fields which don't specify - one. - - .. attribute:: default_output_parser - - The default output parser to use for those fields which don't specify - one. - -.. class:: XPathItemParser([item, selector, response], \**kwargs) - - The :class:`XPathItemParser` class extends the :class:`ItemParser` class - providing more convenient mechanisms for extracting data from web pages - using :ref:`XPath selectors `. - - :class:`XPathItemParser` objects accept two more additional parameters in - their constructors: - - :param selector: The selector to extract data from, when using the - :meth:`add_xpath` or :meth:`replace_xpath` method. - :type selector: :class:`~scrapy.xpath.XPathSelector` object - - :param response: The response used to construct the selector using the - :attr:`default_selector_class`, unless the selector argument is given, - in which case this argument is ignored. - :type response: :class:`~scrapy.http.Response` object - - .. method:: add_xpath(field_name, xpath, re=None) - - Similar to :meth:`ItemParser.add_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`XPathItemParser`. If the ``re`` - argument is given, it's used for extrating data from the selector using - the :meth:`~scrapy.xpath.XPathSelector.re` method. - - :param xpath: the XPath to extract data from - :type xpath: str - - :param re: a regular expression to use for extracting data from the - selected XPath region - :type re: str or compiled regex - - Examples:: - - # HTML snippet:

Color TV

- parser.add_xpath('name', '//p[@class="product-name"]') - # HTML snippet:

the price is $1200

- parser.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - - .. method:: replace_xpath(field_name, xpath, re=None) - - Similar to :meth:`add_xpath` but replaces collected data instead of - adding it. - - .. attribute:: default_selector_class - - The class used to construct the :attr:`selector` of this - :class:`XPathItemParser`, if only a response is given in the constructor. - If a selector is given in the constructor this attribute is ignored. - This attribute is sometimes overridden in subclasses. - - .. attribute:: selector - - The :class:`~scrapy.xpath.XPathSelector` object to extract data from. - It's either the selector given in the constructor or one created from - the response given in the constructor using the - :attr:`default_selector_class`. This attribute is meant to be - read-only. - -.. _topics-itemparser-extending: - -Reusing and extending Item Parsers -================================== - -As your project grows bigger and acquires more and more spiders, maintenance -becomes a fundamental problem, specially when you have to deal with many -different parsing rules for each spider, having a lot of exceptions, but also -wanting to reuse the common parsers. - -Item Parsers are designed to ease the maintenance burden of parsing rules, -without loosing flexibility and, at the same time, providing a convenient -mechanism for extending and overriding them. For this reason Item Parsers -support traditional Python class inheritance for dealing with differences of -specific spiders (or group of spiders). - -Suppose, for example, that some particular site encloses their product names in -three dashes (ie. ``---Plasma TV---``) and you don't want to end up scraping -those dashes in the final product names. - -Here's how you can remove those dashes by reusing and extending the default -Product Item Parser (``ProductParser``):: - - from scrapy.contrib.itemparser.parsers import ApplyConcat - from myproject.itemparsers import ProductParser - - def strip_dashes(x): - return x.strip('-') - - class SiteSpecificParser(ProductParser): - name_in = ApplyConcat(ProductParser.name_in, strip_dashes) - -Another case where extending Item Parsers can be very helpful is when you have -multiple source formats, for example XML and HTML. In the XML version you may -want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - - from scrapy.contrib.itemparser.parsers import ApplyConcat - from myproject.itemparsers import ProductParser - from myproject.utils.xml import remove_cdata - - class XmlProductParser(ProductParser): - name_in = ApplyConcat(remove_cdata, ProductParser.name_in) - -And that's how you typically extend input parsers. - -As for output parsers, it is more common to declare them in the field metadata, -as they usually depend only on the field and not on each specific site parsing -rule (as input parsers do). See also: -:ref:`topics-itemparser-parsers-declaring`. - -There are many other possible ways to extend, inherit and override your Item -Parsers, and different Item Parsers hierarchies may fit better for different -projects. Scrapy only provides the mechanism, it doesn't impose any specific -organization of your Parsers collection - that's up to you and your project -needs. - -.. _topics-itemparser-available-parsers: - -Available built-in parsers -========================== - -Even though you can use any callable function as input and output parsers, -Scrapy provides some commonly used parsers, which are described below. Some of -them, like the :class:`ApplyConcat` (which is typically used as input parser) -composes the output of several functions executed in order, to produce the -final parsed value. - -Here is a list of all built-in parsers: - -.. _topics-itemparser-Applyconcat: - -ApplyConcat parser ------------------- - -The ApplyConcat parser is the recommended parser to use if you want to -concatenate the processing of several functions in a pipeline. - -.. module:: scrapy.contrib.itemparser.parsers - :synopsis: Parser functions to use with Item Parsers - -.. class:: ApplyConcat(\*functions, \**default_parser_context) - - A parser which applies the given functions consecutively, in order, - concatenating their results before next function call. So each function - returns a list of values (though it could return ``None`` or a signle value - too) and the next function is called once for each of those values, - receiving one of those values as input each time. The output of each - function call (for each input value) is concatenated and each values of the - concatenation is used to call the next function, and the process repeats - until there are no functions left. - - Each function can optionally receive a ``parser_context`` parameter, which - will contain the currently active :ref:`Item Parser context - `. - - The keyword arguments passed in the consturctor are used as the default - Item Parser context values passed on each function call. However, the final - Item Parser context values passed to funtions get overriden with the - currently active Item Parser context accesible through the - :meth:`ItemParser.context` attribute. - - Example:: - - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.contrib.itemparser.parsers import ApplyConcat - >>> parser = ApplyConcat(filter_world, str.upper) - >>> parser(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] - -.. class:: TakeFirst - - Return the first non null/empty value from the values to received, so it's - typically used as output parser of single-valued fields. It doesn't receive - any constructor arguments, nor accepts a Item Parser context. - - Example:: - - >>> from scrapy.contrib.itemparser.parsers import TakeFirst - >>> parser = TakeFirst() - >>> parser(['', 'one', 'two', 'three']) - 'one' - -.. class:: Identity - - Return the original values unchanged. It doesn't receive any constructor - arguments nor accepts a Item Parser context. - - Example:: - - >>> from scrapy.contrib.itemparser.parsers import Identity - >>> parser = Identity() - >>> parser(['one', 'two', 'three']) - ['one', 'two', 'three'] - -.. class:: Join(separator=u' ') - - Return the values joined with the separator given in the constructor, which - defaults to ``u' '``. It doesn't accept a Item Parser context. - - When using the default separator, this parser is equivalent to the - function: ``u' '.join`` - - Examples:: - - >>> from scrapy.contrib.itemparser.parsers import Join - >>> parser = Join() - >>> parser(['one', 'two', 'three']) - u'one two three' - >>> parser = Join('
') - >>> parser(['one', 'two', 'three']) - u'one
two
three' diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst new file mode 100644 index 000000000..d96defb71 --- /dev/null +++ b/docs/experimental/loaders.rst @@ -0,0 +1,527 @@ +.. _topics-loaders: + +============ +Item Loaders +============ + +.. module:: scrapy.contrib.loader + :synopsis: Item Loader class + +Item Loaders provide a convenient mechanism for populating scraped :ref:`Items +`. Even though Items can be populated using their own +dictionary-like API, the Item Loaders provide a much more convenient API for +populating them from a scraping process, by automating some common tasks like +parsing the raw extracted data before assigning it. + +In other words, :ref:`Items ` provide the *container* of +scraped data, while Item Loaders provide the mechanism for *populating* that +container. + +Item Loaders are designed to provide a flexible, efficient and easy mechanism +for extending and overriding different field parsing rules, either by spider, +or by source format (HTML, XML, etc) without becoming a nightmare to maintain. + +Using Item Loaders to populate items +==================================== + +To use an Item Loader, you must first instantiate it. You can either +instantiate it with an Item object or without one, in which case an Item is +automatically instantiated in the Item Loader constructor using the Item class +specified in the :attr:`ItemLoader.default_item_class` attribute. + +Then, you start collecting values into the Item Loader, typically using using +:ref:`XPath Selectors `. You can add more than one value to +the same item field, the Item Loader will know how to "join" those values later +using a proper processing function. + +Here is a typical Item Loader usage in a :ref:`Spider `, using +the :ref:`Product item ` declared in the :ref:`Items +chapter `:: + + from scrapy.contrib.loader import XPathItemLoader + from scrapy.xpath import HtmlXPathSelector + from myproject.items import Product + + def parse(self, response): + p = XPathItemLoader(item=Product(), response=response) + p.add_xpath('name', '//div[@class="product_name"]') + p.add_xpath('name', '//div[@class="product_title"]') + p.add_xpath('price', '//p[@id="price"]') + p.add_xpath('stock', '//p[@id="stock"]') + p.add_value('last_updated', 'today') # you can also use literal values + return p.populate_item() + +By quickly looking at that code we can see the ``name`` field is being +extracted from two different XPath locations in the page: + +1. ``//div[@class="product_name"]`` +2. ``//div[@class="product_title"]`` + +In other words, data is being collected by extracting it from two XPath +locations, using the :meth:`~XPathItemLoader.add_xpath` method. This is the data +that will be assigned to the ``name`` field later. + +Afterwards, similar calls are used for ``price`` and ``stock`` fields, and +finally the ``last_update`` field is populated directly with a literal value +(``today``) using a different method: :meth:`~ItemLoader.add_value`. + +Finally, when all data is collected, the :meth:`ItemLoader.populate_item` +method is called which actually populates and returns the item populated with +the data previously extracted and collected with the +:meth:`~XPathItemLoader.add_xpath` and :meth:`~ItemLoader.add_value` calls. + +.. _topics-loaders-processors: + +Input and Output processors +=========================== + +An Item Loader contains one input processor and one output processor for each +(item) field. The input processor processes the extracted data as soon as it's +received (through the :meth:`~XPathItemLoader.add_xpath` or +:meth:`~ItemLoader.add_value` methods) and the result of the input processor is +collected and kept inside the ItemLoader. After collecting all data, the +:meth:`ItemLoader.populate_item` method is called to populate and get the +populated :class:`~scrapy.newitem.Item` object. That's when the output processor +is called with the data previously collected (and processed using the input +processor). The result of the output processor is the final value that gets assigned +to the item. + +Let's see an example to illustrate how this input and output processors are +called for a particular field (the same applies for any other field):: + + p = XPathItemLoader(Product(), some_xpath_selector) + p.add_xpath('name', xpath1) # (1) + p.add_xpath('name', xpath2) # (2) + return p.populate_item() # (3) + +So what happens is: + +1. Data from ``xpath1`` is extracted, and passed through the *input processor* of + the ``name`` field. The result of the input processor is collected and kept in + the Item Loader (but not yet assigned to the item). + +2. Data from ``xpath2`` is extracted, and passed through the same *input + processor* used in (1). The result of the input processor is appended to the + data collected in (1) (if any). + +3. The data collected in (1) and (2) is passed through the *output processor* of + the ``name`` field. The result of the output processor is the value assigned to + the ``name`` field in the item. + +It's worth noticing that processors are just callable objects, which are called +with the data to be parsed, and return a parsed value. So you can use any +function as input or output processor, provided they can receive only one +positional (required) argument. + +The other thing you need to keep in mind is that the values returned by input +processors are collected internally (in lists) and then passed to output +processors to populate the fields, so output processors should expect iterables as +input. + +Last, but not least, Scrapy comes with some :ref:`commonly used processors +` built-in for convenience. + + +Declaring Item Loaders +====================== + +Item Loaders are declared like Items, by using a class definition syntax. Here +is an example:: + + from scrapy.contrib.loader import ItemLoader + from scrapy.contrib.loader.processor import TakeFirst, ApplyConcat, Join + + class ProductLoader(ItemLoader): + + default_input_processor = TakeFirst() + + name_in = ApplyConcat(unicode.title) + name_out = Join() + + price_in = ApplyConcat(unicode.strip) + price_out = TakeFirst() + + # ... + +As you can see, input processors are declared using the ``_in`` suffix while +output processors are declared using the ``_out`` suffix. And you can also +declare a default input/output processors using the +:attr:`ItemLoader.default_input_processor` and +:attr:`ItemLoader.default_output_processor` attributes. + +.. _topics-loaders-processors-declaring: + +Declaring Input and Output Processors +===================================== + +As seen in the previous section, input and output processors can be declared in +the Item Loader definition, and it's very common to declare input processors +this way. However, there is one more place where you can specify the input and +output processors to use: in the :ref:`Item Field ` +metadata. Here is an example:: + + from scrapy.newitem import Item, Field + from scrapy.contrib.loader.processor import ApplyConcat, Join, TakeFirst + + from scrapy.utils.markup import remove_entities + from myproject.utils import filter_prices + + class Product(Item): + name = Field( + input_processor=ApplyConcat(remove_entities), + output_processor=Join(), + ) + price = Field( + default=0, + input_processor=ApplyConcat(remove_entities, filter_prices), + output_processor=TakeFirst(), + ) + +The precedence order, for both input and output processors, is as follows: + +1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most + precedence) +2. Field metadata (``input_processor`` and ``output_processor`` key) +3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and + :meth:`ItemLoader.default_output_processor` (least precedence) + +See also: :ref:`topics-loaders-extending`. + +.. _topics-loaders-context: + +Item Loader Context +=================== + +The Item Loader Context is a dict of arbitrary key/values which is shared among +all input and output processors in the Item Loader. It can be passed when +declaring, instantiating or using Item Loader. They are used to modify the +behaviour of the input/output processors. + +For example, suppose you have a function ``parse_length`` which receives a text +value and extracts a length from it:: + + def parse_length(text, loader_context): + unit = loader_context.get('unit', 'm') + # ... length parsing code goes here ... + return parsed_length + +By accepting a ``loader_context`` argument the function is explicitly telling +the Item Loader that is able to receive an Item Loader context, so the Item +Loader passes the currently active context when calling it, and the processor +function (``parse_length`` in this case) can thus use them. + +There are several ways to modify Item Loader context values: + +1. By modifying the currently active Item Loader context +(:meth:`ItemLoader.context` attribute):: + + loader = ItemLoader(product, unit='cm') + loader.context['unit'] = 'cm' + +2. On Item Loader instantiation (the keyword arguments of Item Loader + constructor are stored in the Item Loader context):: + + p = ItemLoader(product, unit='cm') + +2. On Item Loader declaration, for those input/output processors that support + instatiating them with a Item Loader context. :class:`ApplyConcat` is one of + them:: + + class ProductLoader(ItemLoader): + length_out = ApplyConcat(parse_length, unit='cm') + + +ItemLoader objects +================== + +.. class:: ItemLoader([item], \**kwargs) + + Return a new Item Loader for populating the given Item. If no item is + given, one is instantiated automatically using the class in + :attr:`default_item_class`. + + The item and the remaining keyword arguments are assigned to the Loader + context (accesible through the :attr:`context` attribute). + + .. method:: add_value(field_name, value) + + Add the given ``value`` for the given field. + + The value is passed through the :ref:`field input processor + ` and its result appened to the data + collected for that field. If the field already contains collected data, + the new data is added. + + Examples:: + + loader.add_value('name', u'Color TV') + loader.add_value('colours', [u'white', u'blue']) + loader.add_value('length', u'100', default_unit='cm') + + .. method:: replace_value(field_name, value) + + Similar to :meth:`add_value` but replaces the collected data with the + new value instead of adding it. + + .. method:: populate_item() + + Populate the item with the data collected so far, and return it. The + data collected is first passed through the :ref:`field output processors + ` to get the final value to assign to each + item field. + + .. method:: get_collected_values(field_name) + + Return the collected values for the given field. + + .. method:: get_output_value(field_name) + + Return the collected values parsed using the output processor, for the + given field. This method doesn't populate or modify the item at all. + + .. method:: get_input_processor(field_name) + + Return the input processor for the given field. + + .. method:: get_output_processor(field_name) + + Return the output processor for the given field. + + .. attribute:: item + + The :class:`~scrapy.newitem.Item` object being parsed by this Item + Loader. + + .. attribute:: context + + The currently active :ref:`Context ` of this + Item Loader. + + .. attribute:: default_item_class + + An Item class (or factory), used to instantiate items when not given in + the constructor. + + .. attribute:: default_input_processor + + The default input processor to use for those fields which don't specify + one. + + .. attribute:: default_output_processor + + The default output processor to use for those fields which don't specify + one. + +.. class:: XPathItemLoader([item, selector, response], \**kwargs) + + The :class:`XPathItemLoader` class extends the :class:`ItemLoader` class + providing more convenient mechanisms for extracting data from web pages + using :ref:`XPath selectors `. + + :class:`XPathItemLoader` objects accept two more additional parameters in + their constructors: + + :param selector: The selector to extract data from, when using the + :meth:`add_xpath` or :meth:`replace_xpath` method. + :type selector: :class:`~scrapy.xpath.XPathSelector` object + + :param response: The response used to construct the selector using the + :attr:`default_selector_class`, unless the selector argument is given, + in which case this argument is ignored. + :type response: :class:`~scrapy.http.Response` object + + .. method:: add_xpath(field_name, xpath, re=None) + + Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a + value, which is used to extract a list of unicode strings from the + selector associated with this :class:`XPathItemLoader`. If the ``re`` + argument is given, it's used for extrating data from the selector using + the :meth:`~scrapy.xpath.XPathSelector.re` method. + + :param xpath: the XPath to extract data from + :type xpath: str + + :param re: a regular expression to use for extracting data from the + selected XPath region + :type re: str or compiled regex + + Examples:: + + # HTML snippet:

Color TV

+ loader.add_xpath('name', '//p[@class="product-name"]') + # HTML snippet:

the price is $1200

+ loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') + + .. method:: replace_xpath(field_name, xpath, re=None) + + Similar to :meth:`add_xpath` but replaces collected data instead of + adding it. + + .. attribute:: default_selector_class + + The class used to construct the :attr:`selector` of this + :class:`XPathItemLoader`, if only a response is given in the constructor. + If a selector is given in the constructor this attribute is ignored. + This attribute is sometimes overridden in subclasses. + + .. attribute:: selector + + The :class:`~scrapy.xpath.XPathSelector` object to extract data from. + It's either the selector given in the constructor or one created from + the response given in the constructor using the + :attr:`default_selector_class`. This attribute is meant to be + read-only. + +.. _topics-loaders-extending: + +Reusing and extending Item Loaders +================================== + +As your project grows bigger and acquires more and more spiders, maintenance +becomes a fundamental problem, specially when you have to deal with many +different parsing rules for each spider, having a lot of exceptions, but also +wanting to reuse the common processors. + +Item Loaders are designed to ease the maintenance burden of parsing rules, +without loosing flexibility and, at the same time, providing a convenient +mechanism for extending and overriding them. For this reason Item Loaders +support traditional Python class inheritance for dealing with differences of +specific spiders (or group of spiders). + +Suppose, for example, that some particular site encloses their product names in +three dashes (ie. ``---Plasma TV---``) and you don't want to end up scraping +those dashes in the final product names. + +Here's how you can remove those dashes by reusing and extending the default +Product Item Loader (``ProductLoader``):: + + from scrapy.contrib.loader.processor import ApplyConcat + from myproject.ItemLoaders import ProductLoader + + def strip_dashes(x): + return x.strip('-') + + class SiteSpecificLoader(ProductLoader): + name_in = ApplyConcat(ProductLoader.name_in, strip_dashes) + +Another case where extending Item Loaders can be very helpful is when you have +multiple source formats, for example XML and HTML. In the XML version you may +want to remove ``CDATA`` occurrences. Here's an example of how to do it:: + + from scrapy.contrib.loader.processor import ApplyConcat + from myproject.ItemLoaders import ProductLoader + from myproject.utils.xml import remove_cdata + + class XmlProductLoader(ProductLoader): + name_in = ApplyConcat(remove_cdata, ProductLoader.name_in) + +And that's how you typically extend input processors. + +As for output processors, it is more common to declare them in the field metadata, +as they usually depend only on the field and not on each specific site parsing +rule (as input processors do). See also: +:ref:`topics-loaders-processors-declaring`. + +There are many other possible ways to extend, inherit and override your Item +Loaders, and different Item Loaders hierarchies may fit better for different +projects. Scrapy only provides the mechanism, it doesn't impose any specific +organization of your Loaders collection - that's up to you and your project +needs. + +.. _topics-loaders-available-processors: + +Available built-in processors +============================= + +Even though you can use any callable function as input and output processors, +Scrapy provides some commonly used processors, which are described below. Some +of them, like the :class:`ApplyConcat` (which is typically used as input +processor) composes the output of several functions executed in order, to +produce the final parsed value. + +Here is a list of all built-in processors: + +.. _topics-loaders-applyconcat: + +ApplyConcat processor +--------------------- + +The ApplyConcat processor is the recommended processor to use if you want to +concatenate the processing of several functions in a pipeline. + +.. module:: scrapy.contrib.loader.processor + :synopsis: A collection of processors to use with Item Loaders + +.. class:: ApplyConcat(\*functions, \**default_loader_context) + + A processor which applies the given functions consecutively, in order, + concatenating their results before next function call. So each function + returns a list of values (though it could return ``None`` or a signle value + too) and the next function is called once for each of those values, + receiving one of those values as input each time. The output of each + function call (for each input value) is concatenated and each values of the + concatenation is used to call the next function, and the process repeats + until there are no functions left. + + Each function can optionally receive a ``loader_context`` parameter, which + will contain the currently active :ref:`Item Loader context + `. + + The keyword arguments passed in the consturctor are used as the default + Item Loader context values passed on each function call. However, the final + Item Loader context values passed to funtions get overriden with the + currently active Item Loader context accesible through the + :meth:`ItemLoader.context` attribute. + + Example:: + + >>> def filter_world(x): + ... return None if x == 'world' else x + ... + >>> from scrapy.contrib.loader.processor import ApplyConcat + >>> proc = ApplyConcat(filter_world, str.upper) + >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] + +.. class:: TakeFirst + + Return the first non null/empty value from the values to received, so it's + typically used as output processor of single-valued fields. It doesn't + receive any constructor arguments, nor accepts a Item Loader context. + + Example:: + + >>> from scrapy.contrib.loader.processor import TakeFirst + >>> proc = TakeFirst() + >>> proc(['', 'one', 'two', 'three']) + 'one' + +.. class:: Identity + + Return the original values unchanged. It doesn't receive any constructor + arguments nor accepts a Item Loader context. + + Example:: + + >>> from scrapy.contrib.loader.processor import Identity + >>> proc = Identity() + >>> proc(['one', 'two', 'three']) + ['one', 'two', 'three'] + +.. class:: Join(separator=u' ') + + Return the values joined with the separator given in the constructor, which + defaults to ``u' '``. It doesn't accept a Item Loader context. + + When using the default separator, this processor is equivalent to the + function: ``u' '.join`` + + Examples:: + + >>> from scrapy.contrib.loader.processor import Join + >>> proc = Join() + >>> proc(['one', 'two', 'three']) + u'one two three' + >>> proc = Join('
') + >>> proc(['one', 'two', 'three']) + u'one
two
three' diff --git a/scrapy/contrib/itemparser/common.py b/scrapy/contrib/itemparser/common.py deleted file mode 100644 index a1a8ac069..000000000 --- a/scrapy/contrib/itemparser/common.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Common functions used in Item Parsers code""" - -from functools import partial -from scrapy.utils.python import get_func_args - -def wrap_parser_context(function, context): - """Wrap functions that receive parser_context to contain those parser - arguments pre-loaded and expose a interface that receives only one argument - """ - if 'parser_context' in get_func_args(function): - return partial(function, parser_context=context) - else: - return function diff --git a/scrapy/contrib/itemparser/__init__.py b/scrapy/contrib/loader/__init__.py similarity index 63% rename from scrapy/contrib/itemparser/__init__.py rename to scrapy/contrib/loader/__init__.py index 250453592..fc709c8ce 100644 --- a/scrapy/contrib/itemparser/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -1,7 +1,7 @@ """ -Item Parser +Item Loader -See documentation in docs/topics/itemparser.rst +See documentation in docs/topics/loaders.rst """ from collections import defaultdict @@ -9,14 +9,14 @@ from collections import defaultdict from scrapy.newitem import Item from scrapy.xpath import HtmlXPathSelector from scrapy.utils.misc import arg_to_iter -from .common import wrap_parser_context -from .parsers import Identity +from .common import wrap_loader_context +from .processor import Identity -class ItemParser(object): +class ItemLoader(object): default_item_class = Item - default_input_parser = Identity() - default_output_parser = Identity() + default_input_processor = Identity() + default_output_processor = Identity() def __init__(self, item=None, **context): if item is None: @@ -40,34 +40,34 @@ class ItemParser(object): return item def get_output_value(self, field_name): - parser = self.get_output_parser(field_name) - parser = wrap_parser_context(parser, self.context) - return parser(self._values[field_name]) + proc = self.get_output_processor(field_name) + proc = wrap_loader_context(proc, self.context) + return proc(self._values[field_name]) def get_collected_values(self, field_name): return self._values[field_name] - def get_input_parser(self, field_name): - parser = getattr(self, '%s_in' % field_name, None) - if not parser: - parser = self.item.fields[field_name].get('input_parser', \ - self.default_input_parser) - return parser + def get_input_processor(self, field_name): + proc = getattr(self, '%s_in' % field_name, None) + if not proc: + proc = self.item.fields[field_name].get('input_processor', \ + self.default_input_processor) + return proc - def get_output_parser(self, field_name): - parser = getattr(self, '%s_out' % field_name, None) - if not parser: - parser = self.item.fields[field_name].get('output_parser', \ - self.default_output_parser) - return parser + def get_output_processor(self, field_name): + proc = getattr(self, '%s_out' % field_name, None) + if not proc: + proc = self.item.fields[field_name].get('output_processor', \ + self.default_output_processor) + return proc def _parse_input_value(self, field_name, value): - parser = self.get_input_parser(field_name) - parser = wrap_parser_context(parser, self.context) - return parser(value) + proc = self.get_input_processor(field_name) + proc = wrap_loader_context(proc, self.context) + return proc(value) -class XPathItemParser(ItemParser): +class XPathItemLoader(ItemLoader): default_selector_class = HtmlXPathSelector @@ -79,7 +79,7 @@ class XPathItemParser(ItemParser): selector = self.default_selector_class(response) self.selector = selector context.update(selector=selector, response=response) - super(XPathItemParser, self).__init__(item, **context) + super(XPathItemLoader, self).__init__(item, **context) def add_xpath(self, field_name, xpath, re=None): self.add_value(field_name, self._get_values(field_name, xpath, re)) diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py new file mode 100644 index 000000000..916524947 --- /dev/null +++ b/scrapy/contrib/loader/common.py @@ -0,0 +1,13 @@ +"""Common functions used in Item Loaders code""" + +from functools import partial +from scrapy.utils.python import get_func_args + +def wrap_loader_context(function, context): + """Wrap functions that receive loader_context to contain the context + "pre-loaded" and expose a interface that receives only one argument + """ + if 'loader_context' in get_func_args(function): + return partial(function, loader_context=context) + else: + return function diff --git a/scrapy/contrib/itemparser/parsers.py b/scrapy/contrib/loader/processor.py similarity index 60% rename from scrapy/contrib/itemparser/parsers.py rename to scrapy/contrib/loader/processor.py index f66ce182a..e883c219d 100644 --- a/scrapy/contrib/itemparser/parsers.py +++ b/scrapy/contrib/loader/processor.py @@ -1,26 +1,26 @@ """ -This module provides some commonly used parser functions for Item Parsers. +This module provides some commonly used processors for Item Loaders. -See documentation in docs/topics/itemparser.rst +See documentation in docs/topics/loaders.rst """ from scrapy.utils.misc import arg_to_iter from scrapy.utils.datatypes import MergeDict -from .common import wrap_parser_context +from .common import wrap_loader_context class ApplyConcat(object): - def __init__(self, *functions, **default_parser_context): + def __init__(self, *functions, **default_loader_context): self.functions = functions - self.default_parser_context = default_parser_context + self.default_loader_context = default_loader_context - def __call__(self, value, parser_context=None): + def __call__(self, value, loader_context=None): values = arg_to_iter(value) - if parser_context: - context = MergeDict(parser_context, self.default_parser_context) + if loader_context: + context = MergeDict(loader_context, self.default_loader_context) else: - context = self.default_parser_context - wrapped_funcs = [wrap_parser_context(f, context) for f in self.functions] + context = self.default_loader_context + wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] for func in wrapped_funcs: next_values = [] for v in values: diff --git a/scrapy/tests/test_itemparser.py b/scrapy/tests/test_contrib_loader.py similarity index 62% rename from scrapy/tests/test_itemparser.py rename to scrapy/tests/test_contrib_loader.py index cf0e8addc..d7e36251e 100644 --- a/scrapy/tests/test_itemparser.py +++ b/scrapy/tests/test_contrib_loader.py @@ -1,7 +1,7 @@ import unittest -from scrapy.contrib.itemparser import ItemParser, XPathItemParser -from scrapy.contrib.itemparser.parsers import ApplyConcat, Join, Identity +from scrapy.contrib.loader import ItemLoader, XPathItemLoader +from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity from scrapy.newitem import Item, Field from scrapy.xpath import HtmlXPathSelector from scrapy.http import HtmlResponse @@ -15,30 +15,30 @@ class TestItem(NameItem): url = Field() summary = Field() -# test item parsers +# test item loaders -class NameItemParser(ItemParser): +class NameItemLoader(ItemLoader): default_item_class = TestItem -class TestItemParser(NameItemParser): +class TestItemLoader(NameItemLoader): name_in = ApplyConcat(lambda v: v.title()) -class DefaultedItemParser(NameItemParser): - default_input_parser = ApplyConcat(lambda v: v[:-1]) +class DefaultedItemLoader(NameItemLoader): + default_input_processor = ApplyConcat(lambda v: v[:-1]) -# test parsers +# test processors -def parser_with_args(value, other=None, parser_context=None): - if 'key' in parser_context: - return parser_context['key'] +def processor_with_args(value, other=None, loader_context=None): + if 'key' in loader_context: + return loader_context['key'] return value -class ItemParserTest(unittest.TestCase): +class ItemLoaderTest(unittest.TestCase): def test_populate_item_using_default_loader(self): i = TestItem() i['summary'] = u'lala' - ip = ItemParser(item=i) + ip = ItemLoader(item=i) ip.add_value('name', u'marta') item = ip.populate_item() assert item is i @@ -46,13 +46,13 @@ class ItemParserTest(unittest.TestCase): self.assertEqual(item['name'], [u'marta']) def test_populate_item_using_custom_loader(self): - ip = TestItemParser() + ip = TestItemLoader() ip.add_value('name', u'marta') item = ip.populate_item() self.assertEqual(item['name'], [u'Marta']) def test_add_value(self): - ip = TestItemParser() + ip = TestItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_collected_values('name'), [u'Marta']) self.assertEqual(ip.get_output_value('name'), [u'Marta']) @@ -61,7 +61,7 @@ class ItemParserTest(unittest.TestCase): self.assertEqual(ip.get_output_value('name'), [u'Marta', u'Pepe']) def test_replace_value(self): - ip = TestItemParser() + ip = TestItemLoader() ip.replace_value('name', u'marta') self.assertEqual(ip.get_collected_values('name'), [u'Marta']) self.assertEqual(ip.get_output_value('name'), [u'Marta']) @@ -69,208 +69,208 @@ class ItemParserTest(unittest.TestCase): self.assertEqual(ip.get_collected_values('name'), [u'Pepe']) self.assertEqual(ip.get_output_value('name'), [u'Pepe']) - def test_map_concat_filter(self): + def test_apply_concat_filter(self): def filter_world(x): return None if x == 'world' else x - parser = ApplyConcat(filter_world, str.upper) - self.assertEqual(parser(['hello', 'world', 'this', 'is', 'scrapy']), + proc = ApplyConcat(filter_world, str.upper) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), ['HELLO', 'THIS', 'IS', 'SCRAPY']) def test_map_concat_filter_multiple_functions(self): - class TestItemParser(NameItemParser): + class TestItemLoader(NameItemLoader): name_in = ApplyConcat(lambda v: v.title(), lambda v: v[:-1]) - ip = TestItemParser() + ip = TestItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'Mart']) item = ip.populate_item() self.assertEqual(item['name'], [u'Mart']) - def test_default_input_parser(self): - ip = DefaultedItemParser() + def test_default_input_processor(self): + ip = DefaultedItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'mart']) - def test_inherited_default_input_parser(self): - class InheritDefaultedItemParser(DefaultedItemParser): + def test_inherited_default_input_processor(self): + class InheritDefaultedItemLoader(DefaultedItemLoader): pass - ip = InheritDefaultedItemParser() + ip = InheritDefaultedItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'mart']) - def test_input_parser_inheritance(self): - class ChildItemParser(TestItemParser): + def test_input_processor_inheritance(self): + class ChildItemLoader(TestItemLoader): url_in = ApplyConcat(lambda v: v.lower()) - ip = ChildItemParser() + ip = ChildItemLoader() ip.add_value('url', u'HTTP://scrapy.ORG') self.assertEqual(ip.get_output_value('url'), [u'http://scrapy.org']) ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'Marta']) - class ChildChildItemParser(ChildItemParser): + class ChildChildItemLoader(ChildItemLoader): url_in = ApplyConcat(lambda v: v.upper()) summary_in = ApplyConcat(lambda v: v) - ip = ChildChildItemParser() + ip = ChildChildItemLoader() ip.add_value('url', u'http://scrapy.org') self.assertEqual(ip.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'Marta']) def test_empty_map_concat(self): - class IdentityDefaultedItemParser(DefaultedItemParser): + class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = ApplyConcat() - ip = IdentityDefaultedItemParser() + ip = IdentityDefaultedItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'marta']) - def test_identity_input_parser(self): - class IdentityDefaultedItemParser(DefaultedItemParser): + def test_identity_input_processor(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): name_in = Identity() - ip = IdentityDefaultedItemParser() + ip = IdentityDefaultedItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'marta']) - def test_extend_custom_input_parsers(self): - class ChildItemParser(TestItemParser): - name_in = ApplyConcat(TestItemParser.name_in, unicode.swapcase) + def test_extend_custom_input_processors(self): + class ChildItemLoader(TestItemLoader): + name_in = ApplyConcat(TestItemLoader.name_in, unicode.swapcase) - ip = ChildItemParser() + ip = ChildItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'mARTA']) - def test_extend_default_input_parsers(self): - class ChildDefaultedItemParser(DefaultedItemParser): - name_in = ApplyConcat(DefaultedItemParser.default_input_parser, unicode.swapcase) + def test_extend_default_input_processors(self): + class ChildDefaultedItemLoader(DefaultedItemLoader): + name_in = ApplyConcat(DefaultedItemLoader.default_input_processor, unicode.swapcase) - ip = ChildDefaultedItemParser() + ip = ChildDefaultedItemLoader() ip.add_value('name', u'marta') self.assertEqual(ip.get_output_value('name'), [u'MART']) - def test_output_parser_using_function(self): - ip = TestItemParser() + def test_output_processor_using_function(self): + ip = TestItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) - class TakeFirstItemParser(TestItemParser): + class TakeFirstItemLoader(TestItemLoader): name_out = u" ".join - ip = TakeFirstItemParser() + ip = TakeFirstItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), u'Mar Ta') - def test_output_parser_using_classes(self): - ip = TestItemParser() + def test_output_processor_using_classes(self): + ip = TestItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) - class TakeFirstItemParser(TestItemParser): + class TakeFirstItemLoader(TestItemLoader): name_out = Join() - ip = TakeFirstItemParser() + ip = TakeFirstItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), u'Mar Ta') - class TakeFirstItemParser(TestItemParser): + class TakeFirstItemLoader(TestItemLoader): name_out = Join("
") - ip = TakeFirstItemParser() + ip = TakeFirstItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), u'Mar
Ta') - def test_default_output_parser(self): - ip = TestItemParser() + def test_default_output_processor(self): + ip = TestItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) - class LalaItemParser(TestItemParser): - default_output_parser = Identity() + class LalaItemLoader(TestItemLoader): + default_output_processor = Identity() - ip = LalaItemParser() + ip = LalaItemLoader() ip.add_value('name', [u'mar', u'ta']) self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) - def test_parser_context_on_declaration(self): - class ChildItemParser(TestItemParser): - url_in = ApplyConcat(parser_with_args, key=u'val') + def test_loader_context_on_declaration(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor_with_args, key=u'val') - ip = ChildItemParser() + ip = ChildItemLoader() ip.add_value('url', u'text') self.assertEqual(ip.get_output_value('url'), ['val']) ip.replace_value('url', u'text2') self.assertEqual(ip.get_output_value('url'), ['val']) - def test_parser_context_on_instantiation(self): - class ChildItemParser(TestItemParser): - url_in = ApplyConcat(parser_with_args) + def test_loader_context_on_instantiation(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor_with_args) - ip = ChildItemParser(key=u'val') + ip = ChildItemLoader(key=u'val') ip.add_value('url', u'text') self.assertEqual(ip.get_output_value('url'), ['val']) ip.replace_value('url', u'text2') self.assertEqual(ip.get_output_value('url'), ['val']) - def test_parser_context_on_assign(self): - class ChildItemParser(TestItemParser): - url_in = ApplyConcat(parser_with_args) + def test_loader_context_on_assign(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor_with_args) - ip = ChildItemParser() + ip = ChildItemLoader() ip.context['key'] = u'val' ip.add_value('url', u'text') self.assertEqual(ip.get_output_value('url'), ['val']) ip.replace_value('url', u'text2') self.assertEqual(ip.get_output_value('url'), ['val']) - def test_item_passed_to_input_parser_functions(self): - def parser(value, parser_context): - return parser_context['item']['name'] + def test_item_passed_to_input_processor_functions(self): + def processor(value, loader_context): + return loader_context['item']['name'] - class ChildItemParser(TestItemParser): - url_in = ApplyConcat(parser) + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor) it = TestItem(name='marta') - ip = ChildItemParser(item=it) + ip = ChildItemLoader(item=it) ip.add_value('url', u'text') self.assertEqual(ip.get_output_value('url'), ['marta']) ip.replace_value('url', u'text2') self.assertEqual(ip.get_output_value('url'), ['marta']) def test_add_value_on_unknown_field(self): - ip = TestItemParser() + ip = TestItemLoader() self.assertRaises(KeyError, ip.add_value, 'wrong_field', [u'lala', u'lolo']) -class TestXPathItemParser(XPathItemParser): +class TestXPathItemLoader(XPathItemLoader): default_item_class = TestItem name_in = ApplyConcat(lambda v: v.title()) -class XPathItemParserTest(unittest.TestCase): +class XPathItemLoaderTest(unittest.TestCase): def test_constructor_errors(self): - self.assertRaises(RuntimeError, XPathItemParser) + self.assertRaises(RuntimeError, XPathItemLoader) def test_constructor_with_selector(self): sel = HtmlXPathSelector(text=u"
marta
") - l = TestXPathItemParser(selector=sel) + l = TestXPathItemLoader(selector=sel) self.assert_(l.selector is sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_constructor_with_response(self): response = HtmlResponse(url="", body="
marta
") - l = TestXPathItemParser(response=response) + l = TestXPathItemLoader(response=response) self.assert_(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_output_value('name'), [u'Marta']) def test_add_xpath_re(self): response = HtmlResponse(url="", body="
marta
") - l = TestXPathItemParser(response=response) + l = TestXPathItemLoader(response=response) l.add_xpath('name', '//div/text()', re='ma') self.assertEqual(l.get_output_value('name'), [u'Ma']) From cf566d6238681a8dcfa7ade53801230cb36f44a3 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Wed, 12 Aug 2009 16:56:30 -0300 Subject: [PATCH 4/4] fixed bug with html meta refresh in multiple lines (thanks Molvo for the patch) --- scrapy/tests/test_utils_response.py | 8 ++++++++ scrapy/utils/response.py | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 62ca3be86..1f7c39773 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -54,6 +54,14 @@ class ResponseUtilsTest(unittest.TestCase): response = Response(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + # meta refresh in multiple lines + body = """ + """ + response = Response(url='http://example.org', body=body) + self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage')) + def test_response_httprepr(self): r1 = Response("http://www.example.com") self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 5169b990b..17d204377 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -31,7 +31,7 @@ def get_base_url(response): response.cache['base_url'] = match.group(1) if match else response.url return response.cache['base_url'] -META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.IGNORECASE) +META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE) def get_meta_refresh(response): """ Return a tuple of two strings containing the interval and url included in the http-equiv parameter of the HTML meta element. If no url is included