diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 55468737d..d7eaf261c 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -20,4 +20,4 @@ it's properly merged) . Use at your own risk. :maxdepth: 1 newitems - loaders + itemparser diff --git a/docs/experimental/itemparser.rst b/docs/experimental/itemparser.rst new file mode 100644 index 000000000..160ebe698 --- /dev/null +++ b/docs/experimental/itemparser.rst @@ -0,0 +1,526 @@ +.. _topics-itemparser: + +============ +Item Parsers +============ + +.. module:: scrapy.contrib.itemparser + :synopsis: Item Parser class + +Item Parser provide a convenient mechanism for populating scraped :ref:`Items +`. Even though Items can be populated using their own +dictionary-like API, the Item Parsers provide a much more convenient API for +populating them from a scraping process, by automating some common tasks like +parsing the raw extracted data before assigning it. + +In other words, :ref:`Items ` provide the *container* of +scraped data, while Item Parsers provide the mechanism for *populating* that +container. + +Item Parsers are designed to provide a flexible, efficient and easy mechanism +for extending and overriding different field parsing rules, either by spider, +or by source format (HTML, XML, etc) without becoming a nightmare to maintain. + +Using Item Parsers to populate items +==================================== + +To use an Item Parser, you must first instantiate it. You can either +instantiate it with an Item object or without one, in which case an Item is +automatically instantiated in the Item Parser constructor using the Item class +specified in the :attr:`ItemParser.default_item_class` attribute. + +Then, you start collecting values into the Item Parser, typically using using +:ref:`XPath Selectors `. You can add more than one value to +the same item field, the Item Parser will know how to "join" those values later +using a proper parser function. + +Here is a typical Item Parser usage in a :ref:`Spider `, using +the :ref:`Product item ` declared in the :ref:`Items +chapter `:: + + from scrapy.contrib.itemparser import XPathItemParser + from scrapy.xpath import HtmlXPathSelector + from myproject.items import Product + + def parse(self, response): + p = XPathItemParser(item=Product(), response=response) + p.add_xpath('name', '//div[@class="product_name"]') + p.add_xpath('name', '//div[@class="product_title"]') + p.add_xpath('price', '//p[@id="price"]') + p.add_xpath('stock', '//p[@id="stock"]') + p.add_value('last_updated', 'today') # you can also use literal values + return p.populate_item() + +By quickly looking at that code we can see the ``name`` field is being +extracted from two different XPath locations in the page: + +1. ``//div[@class="product_name"]`` +2. ``//div[@class="product_title"]`` + +In other words, data is being collected by extracting it from two XPath +locations, using the :meth:`~XPathItemParser.add_xpath` method. This is the data +that will be assigned to the ``name`` field later. + +Afterwards, similar calls are used for ``price`` and ``stock`` fields, and +finally the ``last_update`` field is populated directly with a literal value +(``today``) using a different method: :meth:`~ItemParser.add_value`. + +Finally, when all data is collected, the :meth:`ItemParser.populate_item` +method is called which actually populates and returns the item populated with +the data previously extracted and collected with the +:meth:`~XPathItemParser.add_xpath` and :meth:`~ItemParser.add_value` calls. + +.. _topics-itemparser-parsers: + +Input and Output parsers +======================== + +An Item Parser contains one input parser and one output parser for each (item) +field. The input parser processes the extracted data as soon as it's received +(through the :meth:`~XPathItemParser.add_xpath` or +:meth:`~ItemParser.add_value` methods) and the result of the input parser is +collected and kept inside the ItemParser. After collecting all data, the +:meth:`ItemParser.populate_item` method is called to populate and get the +populated :class:`~scrapy.newitem.Item` object. That's when the output parser +is called with the data previously collected (and processed using the input +parser). The result of the output parser is the final value that gets assigned +to the item. + +Let's see an example to illustrate how this input and output parsers are +called for a particular field (the same applies for any other field):: + + p = XPathItemParser(Product(), some_xpath_selector) + p.add_xpath('name', xpath1) # (1) + p.add_xpath('name', xpath2) # (2) + return p.populate_item() # (3) + +So what happens is: + +1. Data from ``xpath1`` is extracted, and passed through the *input parser* of + the ``name`` field. The result of the input parser is collected and kept in + the Item Parser (but not yet assigned to the item). + +2. Data from ``xpath2`` is extracted, and passed through the same *input + parser* used in (1). The result of the input parser is appended to the data + collected in (1) (if any). + +3. The data collected in (1) and (2) is passed through the *output parser* of + the ``name`` field. The result of the output parser is the value assigned to + the ``name`` field in the item. + +It's worth noticing that parsers are just callable objects, which are called +with the data to be parsed, and return a parsed value. So you can use any +function as input or output parser, provided they can receive only one +positional (required) argument. + +The other thing you need to keep in mind is that the values returned by input +parsers are collected internally (in lists) and then passed to output parsers +to populate the fields, so output parsers should expect iterables as input. + +Last, but not least, Scrapy comes with some :ref:`commonly used parsers +` built-in for convenience. + + +Declaring Item Parsers +====================== + +Item Parsers are declared like Items, by using a class definition syntax. Here +is an example:: + + from scrapy.contrib.itemparser import ItemParser + from scrapy.contrib.itemparser.parsers import TakeFirst, ApplyConcat, Join + + class ProductParser(ItemParser): + + default_expander = TakeFirst() + + name_in = ApplyConcat(unicode.title) + name_out = Join() + + price_in = ApplyConcat(unicode.strip) + price_out = TakeFirst() + + # ... + +As you can see, input parsers are declared using the ``_in`` suffix while +output parsers are declared using the ``_out`` suffix. And you can also declare +a default input/output parsers using the +:attr:`ItemParser.default_input_parser` and +:attr:`ItemParser.default_output_parser` attributes. + +.. _topics-itemparser-parsers-declaring: + +Declaring Input and Output Parsers +================================== + +As seen in the previous section, input and output parsers can be declared in +the Item Parser definition, and it's very common to declare input parsers this +way. However, there is one more place where you can specify the input and +output parsers to use: in the :ref:`Item Field ` +metadata. Here is an example:: + + from scrapy.newitem import Item, Field + from scrapy.contrib.itemparser.parser import ApplyConcat, Join, TakeFirst + + from scrapy.utils.markup import remove_entities + from myproject.utils import filter_prices + + class Product(Item): + name = Field( + input_parser=ApplyConcat(remove_entities), + output_parser=Join(), + ) + price = Field( + default=0, + input_parser=ApplyConcat(remove_entities, filter_prices), + output_parser=TakeFirst(), + ) + +The precedence order, for both input and output parsers, is as follows: + +1. Item Parser field-specific attributes: ``field_in`` and ``field_out`` (most + precedence) +2. Field metadata (``input_parser`` and ``output_parser`` key) +3. Item Parser defaults: :meth:`ItemParser.default_expander` and + :meth:`ItemParser.default_output_parser` (least precedence) + +See also: :ref:`topics-itemparser-extending`. + +.. _topics-itemparser-context: + +Item Parser Context +=================== + +The Item Parser Context is a dict of arbitrary key/values which is shared among +all input and output parsers in the Item Parser. It can be passed when +declaring, instantiating or using Item Parser. They are used to modify the +behaviour of the input/output parsers. + +For example, suppose you have a function ``parse_length`` which receives a text +value and extracts a length from it:: + + def parse_length(text, parser_context): + unit = parser_context.get('unit', 'm') + # ... length parsing code goes here ... + return parsed_length + +By accepting a ``parser_context`` argument the function is explicitly telling +the Item Parser that is able to receive an Item Parser context, so the Item +Parser passes the currently active context when calling it, and the parser +function (``parse_length`` in this case) can thus use them. + +There are several ways to modify Item Parser context values: + +1. By modifying the currently active Item Parser context +(:meth:`ItemParser.context` attribute):: + + parser = ItemParser(product, unit='cm') + parser.context['unit'] = 'cm' + +2. On Item Parser instantiation (the keyword arguments of Item Parser + constructor are stored in the Item Parser context):: + + p = ItemParser(product, unit='cm') + +2. On Item Parser declaration, for those input/output parsers that support + instatiating them with a Item Parser context. :class:`ApplyConcat` is one of + them:: + + class ProductParser(ItemParser): + length_out = ApplyConcat(parse_length, unit='cm') + + +ItemParser objects +================== + +.. class:: ItemParser([item], \**kwargs) + + Return a new Item Parser for populating the given Item. If no item is + given, one is instantiated automatically using the class in + :attr:`default_item_class`. + + The item and the remaining keyword arguments are assigned to the Parser + context (accesible through the :attr:`context` attribute). + + .. method:: add_value(field_name, value) + + Add the given ``value`` for the given field. + + The value is passed through the :ref:`field input parser + ` and its result appened to the data + collected for that field. If the field already contains collected data, + the new data is added. + + Examples:: + + parser.add_value('name', u'Color TV') + parser.add_value('colours', [u'white', u'blue']) + parser.add_value('length', u'100', default_unit='cm') + + .. method:: replace_value(field_name, value) + + Similar to :meth:`add_value` but replaces the collected data with the + new value instead of adding it. + + .. method:: populate_item() + + Populate the item with the data collected so far, and return it. The + data collected is first passed through the :ref:`field output parsers + ` to get the final value to assign to each + item field. + + .. method:: get_collected_values(field_name) + + Return the collected values for the given field. + + .. method:: get_output_value(field_name) + + Return the collected values parsed using the output parser, for the + given field. This method doesn't populate or modify the item at all. + + .. method:: get_input_parser(field_name) + + Return the input parser for the given field. + + .. method:: get_output_parser(field_name) + + Return the output parser for the given field. + + .. attribute:: item + + The :class:`~scrapy.newitem.Item` object being parsed by this Item + Parser. + + .. attribute:: context + + The currently active :ref:`Context ` of this + Item Parser. + + .. attribute:: default_item_class + + An Item class (or factory), used to instantiate items when not given in + the constructor. + + .. attribute:: default_input_parser + + The default input parser to use for those fields which don't specify + one. + + .. attribute:: default_output_parser + + The default output parser to use for those fields which don't specify + one. + +.. class:: XPathItemParser([item, selector, response], \**kwargs) + + The :class:`XPathItemParser` class extends the :class:`ItemParser` class + providing more convenient mechanisms for extracting data from web pages + using :ref:`XPath selectors `. + + :class:`XPathItemParser` objects accept two more additional parameters in + their constructors: + + :param selector: The selector to extract data from, when using the + :meth:`add_xpath` or :meth:`replace_xpath` method. + :type selector: :class:`~scrapy.xpath.XPathSelector` object + + :param response: The response used to construct the selector using the + :attr:`default_selector_class`, unless the selector argument is given, + in which case this argument is ignored. + :type response: :class:`~scrapy.http.Response` object + + .. method:: add_xpath(field_name, xpath, re=None) + + Similar to :meth:`ItemParser.add_value` but receives an XPath instead of a + value, which is used to extract a list of unicode strings from the + selector associated with this :class:`XPathItemParser`. If the ``re`` + argument is given, it's used for extrating data from the selector using + the :meth:`~scrapy.xpath.XPathSelector.re` method. + + :param xpath: the XPath to extract data from + :type xpath: str + + :param re: a regular expression to use for extracting data from the + selected XPath region + :type re: str or compiled regex + + Examples:: + + # HTML snippet:

Color TV

+ parser.add_xpath('name', '//p[@class="product-name"]') + # HTML snippet:

the price is $1200

+ parser.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') + + .. method:: replace_xpath(field_name, xpath, re=None) + + Similar to :meth:`add_xpath` but replaces collected data instead of + adding it. + + .. attribute:: default_selector_class + + The class used to construct the :attr:`selector` of this + :class:`XPathItemParser`, if only a response is given in the constructor. + If a selector is given in the constructor this attribute is ignored. + This attribute is sometimes overridden in subclasses. + + .. attribute:: selector + + The :class:`~scrapy.xpath.XPathSelector` object to extract data from. + It's either the selector given in the constructor or one created from + the response given in the constructor using the + :attr:`default_selector_class`. This attribute is meant to be + read-only. + +.. _topics-itemparser-extending: + +Reusing and extending Item Parsers +================================== + +As your project grows bigger and acquires more and more spiders, maintenance +becomes a fundamental problem, specially when you have to deal with many +different parsing rules for each spider, having a lot of exceptions, but also +wanting to reuse the common parsers. + +Item Parsers are designed to ease the maintenance burden of parsing rules, +without loosing flexibility and, at the same time, providing a convenient +mechanism for extending and overriding them. For this reason Item Parsers +support traditional Python class inheritance for dealing with differences of +specific spiders (or group of spiders). + +Suppose, for example, that some particular site encloses their product names in +three dashes (ie. ``---Plasma TV---``) and you don't want to end up scraping +those dashes in the final product names. + +Here's how you can remove those dashes by reusing and extending the default +Product Item Parser (``ProductParser``):: + + from scrapy.contrib.itemparser.parsers import ApplyConcat + from myproject.itemparsers import ProductParser + + def strip_dashes(x): + return x.strip('-') + + class SiteSpecificParser(ProductParser): + name_in = ApplyConcat(ProductParser.name_in, strip_dashes) + +Another case where extending Item Parsers can be very helpful is when you have +multiple source formats, for example XML and HTML. In the XML version you may +want to remove ``CDATA`` occurrences. Here's an example of how to do it:: + + from scrapy.contrib.itemparser.parsers import ApplyConcat + from myproject.itemparsers import ProductParser + from myproject.utils.xml import remove_cdata + + class XmlProductParser(ProductParser): + name_in = ApplyConcat(remove_cdata, ProductParser.name_in) + +And that's how you typically extend input parsers. + +As for output parsers, it is more common to declare them in the field metadata, +as they usually depend only on the field and not on each specific site parsing +rule (as input parsers do). See also: +:ref:`topics-itemparser-parsers-declaring`. + +There are many other possible ways to extend, inherit and override your Item +Parsers, and different Item Parsers hierarchies may fit better for different +projects. Scrapy only provides the mechanism, it doesn't impose any specific +organization of your Parsers collection - that's up to you and your project +needs. + +.. _topics-itemparser-available-parsers: + +Available built-in parsers +========================== + +Even though you can use any callable function as input and output parsers, +Scrapy provides some commonly used parsers, which are described below. Some of +them, like the :class:`ApplyConcat` (which is typically used as input parser) +composes the output of several functions executed in order, to produce the +final parsed value. + +Here is a list of all built-in parsers: + +.. _topics-itemparser-Applyconcat: + +ApplyConcat parser +------------------ + +The ApplyConcat parser is the recommended parser to use if you want to +concatenate the processing of several functions in a pipeline. + +.. module:: scrapy.contrib.itemparser.parsers + :synopsis: Parser functions to use with Item Parsers + +.. class:: ApplyConcat(\*functions, \**default_parser_context) + + A parser which applies the given functions consecutively, in order, + concatenating their results before next function call. So each function + returns a list of values (though it could return ``None`` or a signle value + too) and the next function is called once for each of those values, + receiving one of those values as input each time. The output of each + function call (for each input value) is concatenated and each values of the + concatenation is used to call the next function, and the process repeats + until there are no functions left. + + Each function can optionally receive a ``parser_context`` parameter, which + will contain the currently active :ref:`Item Parser context + `. + + The keyword arguments passed in the consturctor are used as the default + Item Parser context values passed on each function call. However, the final + Item Parser context values passed to funtions get overriden with the + currently active Item Parser context accesible through the + :meth:`ItemParser.context` attribute. + + Example:: + + >>> def filter_world(x): + ... return None if x == 'world' else x + ... + >>> from scrapy.contrib.itemparser.parsers import ApplyConcat + >>> parser = ApplyConcat(filter_world, str.upper) + >>> parser(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] + +.. class:: TakeFirst + + Return the first non null/empty value from the values to received, so it's + typically used as output parser of single-valued fields. It doesn't receive + any constructor arguments, nor accepts a Item Parser context. + + Example:: + + >>> from scrapy.contrib.itemparser.parsers import TakeFirst + >>> parser = TakeFirst() + >>> parser(['', 'one', 'two', 'three']) + 'one' + +.. class:: Identity + + Return the original values unchanged. It doesn't receive any constructor + arguments nor accepts a Item Parser context. + + Example:: + + >>> from scrapy.contrib.itemparser.parsers import Identity + >>> parser = Identity() + >>> parser(['one', 'two', 'three']) + ['one', 'two', 'three'] + +.. class:: Join(separator=u' ') + + Return the values joined with the separator given in the constructor, which + defaults to ``u' '``. It doesn't accept a Item Parser context. + + When using the default separator, this parser is equivalent to the + function: ``u' '.join`` + + Examples:: + + >>> from scrapy.contrib.itemparser.parsers import Join + >>> parser = Join() + >>> parser(['one', 'two', 'three']) + u'one two three' + >>> parser = Join('
') + >>> parser(['one', 'two', 'three']) + u'one
two
three' diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst deleted file mode 100644 index dfec482d2..000000000 --- a/docs/experimental/loaders.rst +++ /dev/null @@ -1,501 +0,0 @@ -.. _topics-loader: - -============ -Item Loaders -============ - -.. module:: scrapy.newitem.loader - :synopsis: Item Loader class - -Item Loaders (or Loaders, for short) provide a convenient mechanism for -populating scraped :ref:`Items `. Even though Items can be -populated using their own dictionary-like API, the Loaders provide a much more -convenient API for populating them from a scraping process, by automating some -common tasks like parsing the raw extracted data before assigning it. - -In other words, :ref:`Items ` provide the *container* of -scraped data, while Loaders provide the mechanism for *populating* that -container. - -Loaders are designed to provide a flexible, efficient and easy mechanism for -extending and overriding different field parsing rules (either by spider, or by -source format) without becoming a nightmare to maintain - -Using Loaders to populate items -=============================== - -To use a Loader, you must first instantiate it. You can either instantiate it -with an Item object or without one, in which case an Item is automatically -instantiated in the Loader constructor using the Item class specified in the -:attr:`Loader.default_item_class` attribute. - -Then, you start adding values to the Loader, typically collecting them using -:ref:`Selectors `. You can add more than one value to the -same item field, the Loader will know how to "join" those values later using a -Reducer. - -Here is a typical Loader usage in a :ref:`Spider `, using the -:ref:`Product item ` declared in the :ref:`Items -section `:: - - from scrapy.item.loader import XPathLoader - from scrapy.xpath import HtmlXPathSelector - from myproject.items import Product - - def parse(self, response): - l = XPathLoader(item=Product(), response=response) - l.add_xpath('name', '//div[@class="product_name"]') - l.add_xpath('name', '//div[@class="product_title"]') - l.add_xpath('price', '//p[@id="price"]') - l.add_xpath('stock', '//p[@id="stock"]') - l.add_value('last_updated', 'today') # you can also use literal values - return l.get_item() - -By quickly looking at that code we can see the ``name`` field is being -extracted from two different XPath locations in the page: - -1. ``//div[@class="product_name"]`` -2. ``//div[@class="product_title"]`` - -In other words, data is being collected by extracting it from two XPath -locations, using the :meth:`~XPathLoader.add_xpath` method. This is the data -that will be assigned to the ``name`` field later. - -Afterwards, similar calls are used for ``price`` and ``stock`` fields, and -finally the ``last_update`` field is populated directly with a literal value -(``today``) using a different method: :meth:`~Loader.add_value`. - -Finally, when all data is collected, the :meth:`Loader.get_item` method is -called which actually populates and returns the item populated with the data -previously extracted and collected with the :meth:`~XPathLoader.add_xpath` and -:meth:`~Loader.add_value` calls. - -.. _topics-loader-expred: - -Expanders and Reducers -====================== - -A Loader is composed of one expander and one reducer for each (item) field. The -Expander processes the extracted data as soon as it's received (through the -:meth:`~XPathLoader.add_xpath` or :meth:`~Loader.add_value` methods) and the -result of the expander is collected and kept inside the Loader. After -collecting all data, the :meth:`Loader.get_item` method is called to actually -populate and get the :class:`~scrapy.newitem.Item` object. That's when the -Reducers are called with the data previously collected (using the Expanders) -and the output of the Reducers are the actual values that get assigned to the -item. - -Let's see an example to illustrate how Expanders and Reducers are called, for a -particular field (the same applies for any other field):: - - l = XPathLoader(Product(), some_selector) - l.add_xpath('name', xpath1) # (1) - l.add_xpath('name', xpath2) # (2) - return l.get_item() # (3) - -So what happens is: - -1. Data from ``xpath1`` is extracted, and passed through the Expander of the - ``name`` field. The output of the expander is collected and kept in the - loader (but not yet assigned to the item). - -2. Data from ``xpath2`` is extracted, and passed through the same Expander used - in (1). The output of the expander is appended to the data collected in (1) - (if any). - -3. The data collected in (1) and (2) is passed through the Reducer of the - ``name`` field. The output of the Reducer is the value assigned to the - ``name`` field in the item. - -Scrapy comes with one major expander built-in, the :ref:`Tree Expander -`, and :ref:`a couple of commonly used reducers -`. - -Declaring Loaders -================= - -Loaders are declared like Items, by using a class definition syntax. Here is an -example:: - - from scrapy.newitem.loader import Loader - from scrapy.newitem.loader.expanders import TreeExpander - from scrapy.newitem.loader.reducers import Join, TakeFirst - - class ProductLoader(Loader): - - default_expander = TakeFirst() - - name_exp = TreeExpander(unicode.title) - name_red = Join() - - price_exp = TreeExpander(unicode.strip) - price_red = TakeFirst() - - ... - -As you can see, expanders are declared using the ``_exp`` suffix while reducers -are declared using the ``_red`` suffix. And you can also declare a default -expander using the :attr:`Loader.default_expander` attribute. - -.. _topics-loader-expred-declaring: - -Declaring Extenders and Reducers -================================ - -As seen in the previous section, extenders and reducers can be declared in the -Loader definition, and it's very common to declare expanders this way. However, -there is one more place where you can specify the exanders and reducers to use: -in the :ref:`Item Field ` metadata. Here is an example:: - - from scrapy.newitem import Item, Field - from scrapy.newitem.loader.expanders import TreeExpander - from scrapy.newitem.loader.reducers import Join, TakeFirst - - from scrapy.utils.markup import remove_entities - from myprojct.utils import filter_prices - - class Product(Item): - name = Field( - expander=TreeExpander(remove_entities), - reducer=Join(), - ) - price = Field( - default=0, - expander=TreeExpander(remove_entities, filter_prices), - reducer=TakeFirst(), - ) - -The precendece order, for both expander and reducer declarations, is as -follows: - -1. Loader field-specific attributes: ``field_exp`` and ``field_red`` (more - precedence) -2. Field metadata (``expander`` and ``reducer`` key) -3. Loader defaults: :meth:`Loader.default_expander` and - :meth:`Loader.default_reducer` (less precedence) - -See also: :ref:`topics-loader-extending`. - -.. _topics-loader-args: - -Item Loader arguments -===================== - -The Loader arguments is a dict of arbitrary key/values which can be passed when -declaring, instantiating or using Loaders. They are used modify the behaviour -of the expanders. - -For example, suppose you have a function ``parse_length`` which receives a text -value and extracts a length from it:: - - def parse_length(text, loader_args): - unit = loader_args('unit', 'm') - # ... length parsing code goes here ... - return parsed_length - -Since it receives a ``loader_args`` the Expander will pass the currently active -Loader arguments when calling it. - -There are seveal ways to pass Loader arguments: - -1. Passing arguments on Loader declaration:: - - class ProductLoader(Loader): - length_exp = TreeExpander(parse_length, unit='cm') - -2. Passing arguments on Loader instantiation:: - - l = Loader(product, unit='cm') - -3. Passing arguments on Loader usage:: - - l.add_xpath('length', '//div', unit='cm') - -Loader objects -============== - -.. class:: Loader([item], \**loader_args) - - Return a new Item Loader for populating the given Item. If no item is - given, one is instantiated using the class in :attr:`default_item_class`. - - .. method:: add_value(field_name, value, \**new_loader_args) - - Add the given ``value`` for the given field. - - The value is passed through the :ref:`field expander - ` and its output appened to the data collected - for that field. If the field already contains collected data, the new - data is added. - - If any keyword arguments are passed, they're used as :ref:`Loader - arguments ` when calling the expanders. - - Examples:: - - loader.add_value('name', u'Color TV') - loader.add_value('colours', [u'white', u'blue']) - loader.add_value('length', u'100', default_unit='cm') - - .. method:: replace_value(field_name, value, \**new_loader_args) - - Similar to :meth:`add_value` but replaces collected data instead of - adding it. - - - .. method:: get_item() - - Populate the item with the data collected so far, and return it. The - data collected is first passed through the :ref:`field reducers - ` to get the final value to assign to each item - field. - - .. method:: get_expanded_value(field_name) - - Return the expanded data for the given field. In other words, return - the dat collected so far for the given field, without reducing it. - - .. method:: get_reduced_value(field_name) - - Return the reduced value for the given field, without modifying the - item. - - .. method:: get_expander(field_name) - - Return the expander for the given field. - - .. method:: get_reducer(field_name) - - Return the reducer for the given field. - - .. attribute:: default_item_class - - An Item class (or factory), used to instantiate items when not given in - the constructor. - - .. attribute:: default_expander - - The default expander to use for those fields which don't define a - specific expander - - .. attribute:: default_reducer - - The default reducer to use for those fields which don't define a - specific expander - -.. class:: XPathLoader([item, selector, response], \**loader_args) - - The :class:`XPathLoader` class extends the :class:`Loader` class providing - more convenient mechanisms for extracting data from web pages using - :ref:`XPath selectors `. - - :class:`XPathLoader` objects accept two more additional parameters in their - constructors: - - :param selector: The selector to extract data from, when using the - :meth:`add_xpath` or :meth:`replace_xpath` method. - :type selector: :class:`~scrapy.xpath.XPathSelector` object - - :param response: The response used to construct the selector using the - :attr:`default_selector_class`, unless the selector argument is given, - in which case this argument is ignored. - :type response: :class:`~scrapy.http.Response` object - - .. method:: add_xpath(field_name, xpath, re=None, \**new_loader_args) - - Similar to :meth:`Loader.add_value` but receives an XPath instead of a - value, which is used to extract a list of unicode strings from the - selector associated with this :class:`XPathLoader`. If the ``re`` - argument is given, it's used for extrating data from the selector using - the :meth:`~scrapy.xpath.XPathSelector.re` method. - - :param xpath: the XPath to extract data from - :type xpath: str - - :param re: a regular expression to use for extracting data from the - selected XPath region - :type re: str or compiled regex - - Examples:: - - # HTML snippet:

Color TV

- loader.add_xpath('name', '//p[@class="product-name"]') - # HTML snippet:

the price is $1200

- loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - - .. method:: replace_xpath(field_name, xpath, re=None, \**new_loader_args) - - Similar to :meth:`add_xpath` but replaces collected data instead of - adding it. - - .. attribute:: default_selector_class - - The class used to construct the :attr:`selector` of this - :class:`XPathLoader`, if only a response is given in the constructor. - If a selector is given in the constructor this attribute is ignored. - This attribute is sometimes overridden in subclasses. - - .. attribute:: selector - - The :class:`~scrapy.xpath.XPathSelector` object to extract data from. - It's either the selector given in the constructor or one created from - the response given in the constructor using the - :attr:`default_selector_class`. This attribute is meant to be - read-only. - -.. _topics-loader-extending: - -Reusing and extending Loaders -============================= - -As your project grows bigger and acquires more and more spiders, maintenance -becomes a fundamental problem, specially when you have to deal with many -different parsing rules per spider, a lot of exceptions, but also want to reuse -the common cases. - -Loaders are designed to ease the maintenance of parsing rules, without loosing -flexibility and, at the same time, providing a convenient mechanism for -extending and overriding them. For this reason Loaders support traditional -class inheritance for for dealing with differences of specific spiders (or -group of spiders). - -Suppose, for example, that some particular site encloses their product names -between three dashes (ie. ``---Plasma TV---``) and you don't want to end up -scraping those dashes in the final product names. - -Here's how you can remove those dashes by reusing and extending the default -Product Loader:: - - strip_dashes = lambda x: x.strip('-') - - class SiteSpecificLoader(ProductLoader): - name_exp = TreeExpander(ProductLoader.name_exp, strip_dashes) - -Another case where extending Loaders can be very helpful is when you have -multiple source formats, for example XML and HTML. In the XML version you may -want to remove ``CDATA`` occurrences. Here's an example of how to do it:: - - from myproject.utils.xml import remove_cdata - - class XmlLoader(ProductLoader): - name_exp = TreeExpander(remove_cdata, ProductLoader.name_exp) - -And that's how you typically extend expanders. - -As for reducers, it is more common to declare them in the field metadata, as -they usually depend only on the field and not on each specific site parsing -rule (as expanders do). See also: :ref:`topics-loader-expred-declaring`. - -There are many other possible ways to extend, inherit and override your -Loaders, and different Loader hierarchies may fit better for different -projects. Scrapy only provides the mechanism, it doesn't impose any specific -organization of your Loaders collection - that's up to you and your project -needs. - -Available Expanders -=================== - -.. _topics-loader-tree-expander: - -Tree Expander -------------- - -The Tree Expander is the recommended Expander to use and the only really useful -one, as the other is just an identity expander. - -.. module:: scrapy.newitem.loader.expanders - :synopsis: Expander classes to use with Item Loaders - -.. class:: TreeExpander(\*functions, \**default_loader_arguments) - - An expander which applies the given functions consecutively, in order, to - each value returned by the previous function. - - The algorithm consists in an ordered list of functions, each of which - receives one value and can return zero, one or more values (as a list or - iterable). If a function returns more than one value, the next function in - the list will be called with each of those values, potentially returning - more values and thus expanding the execution into different branches, which - is why this expander is called Tree Expander. - - Each expander function can optionally receive a ``loader_args`` argument, - which will contain the currently active :ref:`Loader arguments - `. - - The keyword arguments passed in the consturctor are used as the default - Loader arguments passed to on each expander call. This arguments can be - overriden with specific noader arguments passed on each expander call. - - Example:: - - >>> def filter_world(x): - ... return None if x == 'world' else x - ... - >>> from scrapy.newitem.loader.expanders import TreeExpander - >>> expander = TreeExpander(filter_world, str.upper) - >>> expander(['hello', 'world', 'this', 'is', 'scrapy']) - ['HELLO, 'THIS', 'IS', 'SCRAPY'] - - -IdentityExpander ----------------- - -.. class:: IdentityExpander - - An expander which returns the original values unchanged. It doesn't support - any constructor arguments. - -.. _topics-loader-reducers: - -Available Reducers -================== - -.. module:: scrapy.newitem.loader.reducers - :synopsis: Reducer classes to use with Item Loaders - -Reducers are callable objects which are called with a list of values (to be -reduced) as their first and only argument. Scrapy provides some simple, -commonly used reducers, which are described below. But you can use any function -or callable as reducer. - -.. class:: TakeFirst - - Return the first non-null value from the values to reduce, so it's used for - single-valued fields. It doesn't receive any constructor arguments. - - Example:: - - >>> from scrapy.newitem.loader.reducers import TakeFirst - >>> reducer = TakeFirst() - >>> reducer(['', 'one', 'two', 'three']) - 'one' - -.. class:: Identity - - Return the values to reduce unchanged, so it's used for multi-valued - fields. It doesn't receive any constructor arguments. - - Example:: - - >>> from scrapy.newitem.loader.reducers import Identity - >>> reducer = Identity() - >>> reducer(['one', 'two', 'three']) - ['one', 'two', 'three'] - -.. class:: Join(separator=u' ') - - Return a the values to reduce joined with the separator given in the - constructor, which defaults to ``u' '``. - - When using the default separator, this reducer is equivalent to the - function: ``u' '.join`` - - Examples:: - - >>> from scrapy.newitem.loader.reducers import Join - >>> reducer = Join() - >>> reducer(['one', 'two', 'three']) - u'one two three' - >>> reducer = Join('
') - >>> reducer(['one', 'two', 'three']) - u'one
two
three' diff --git a/scrapy/contrib/itemparser/__init__.py b/scrapy/contrib/itemparser/__init__.py new file mode 100644 index 000000000..250453592 --- /dev/null +++ b/scrapy/contrib/itemparser/__init__.py @@ -0,0 +1,93 @@ +""" +Item Parser + +See documentation in docs/topics/itemparser.rst +""" + +from collections import defaultdict + +from scrapy.newitem import Item +from scrapy.xpath import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter +from .common import wrap_parser_context +from .parsers import Identity + +class ItemParser(object): + + default_item_class = Item + default_input_parser = Identity() + default_output_parser = Identity() + + def __init__(self, item=None, **context): + if item is None: + item = self.default_item_class() + self.item = context['item'] = item + self.context = context + self._values = defaultdict(list) + + def add_value(self, field_name, value): + parsed_value = self._parse_input_value(field_name, value) + self._values[field_name] += arg_to_iter(parsed_value) + + def replace_value(self, field_name, value): + parsed_value = self._parse_input_value(field_name, value) + self._values[field_name] = arg_to_iter(parsed_value) + + def populate_item(self): + item = self.item + for field_name in self._values: + item[field_name] = self.get_output_value(field_name) + return item + + def get_output_value(self, field_name): + parser = self.get_output_parser(field_name) + parser = wrap_parser_context(parser, self.context) + return parser(self._values[field_name]) + + def get_collected_values(self, field_name): + return self._values[field_name] + + def get_input_parser(self, field_name): + parser = getattr(self, '%s_in' % field_name, None) + if not parser: + parser = self.item.fields[field_name].get('input_parser', \ + self.default_input_parser) + return parser + + def get_output_parser(self, field_name): + parser = getattr(self, '%s_out' % field_name, None) + if not parser: + parser = self.item.fields[field_name].get('output_parser', \ + self.default_output_parser) + return parser + + def _parse_input_value(self, field_name, value): + parser = self.get_input_parser(field_name) + parser = wrap_parser_context(parser, self.context) + return parser(value) + + +class XPathItemParser(ItemParser): + + default_selector_class = HtmlXPathSelector + + def __init__(self, item=None, selector=None, response=None, **context): + if selector is None and response is None: + raise RuntimeError("%s must be instantiated with a selector" \ + "or response" % self.__class__.__name__) + if selector is None: + selector = self.default_selector_class(response) + self.selector = selector + context.update(selector=selector, response=response) + super(XPathItemParser, self).__init__(item, **context) + + def add_xpath(self, field_name, xpath, re=None): + self.add_value(field_name, self._get_values(field_name, xpath, re)) + + def replace_xpath(self, field_name, xpath, re=None): + self.replace_value(field_name, self._get_values(field_name, xpath, re)) + + def _get_values(self, field_name, xpath, re): + x = self.selector.x(xpath) + return x.re(re) if re else x.extract() + diff --git a/scrapy/contrib/itemparser/common.py b/scrapy/contrib/itemparser/common.py new file mode 100644 index 000000000..a1a8ac069 --- /dev/null +++ b/scrapy/contrib/itemparser/common.py @@ -0,0 +1,13 @@ +"""Common functions used in Item Parsers code""" + +from functools import partial +from scrapy.utils.python import get_func_args + +def wrap_parser_context(function, context): + """Wrap functions that receive parser_context to contain those parser + arguments pre-loaded and expose a interface that receives only one argument + """ + if 'parser_context' in get_func_args(function): + return partial(function, parser_context=context) + else: + return function diff --git a/scrapy/contrib/itemparser/parsers.py b/scrapy/contrib/itemparser/parsers.py new file mode 100644 index 000000000..f66ce182a --- /dev/null +++ b/scrapy/contrib/itemparser/parsers.py @@ -0,0 +1,52 @@ +""" +This module provides some commonly used parser functions for Item Parsers. + +See documentation in docs/topics/itemparser.rst +""" + +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.datatypes import MergeDict +from .common import wrap_parser_context + +class ApplyConcat(object): + + def __init__(self, *functions, **default_parser_context): + self.functions = functions + self.default_parser_context = default_parser_context + + def __call__(self, value, parser_context=None): + values = arg_to_iter(value) + if parser_context: + context = MergeDict(parser_context, self.default_parser_context) + else: + context = self.default_parser_context + wrapped_funcs = [wrap_parser_context(f, context) for f in self.functions] + for func in wrapped_funcs: + next_values = [] + for v in values: + next_values += arg_to_iter(func(v)) + values = next_values + return list(values) + + +class TakeFirst(object): + + def __call__(self, values): + for value in values: + if value: + return value + + +class Identity(object): + + def __call__(self, values): + return values + + +class Join(object): + + def __init__(self, separator=u' '): + self.separator = separator + + def __call__(self, values): + return self.separator.join(values) diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py deleted file mode 100644 index f40e4c7e5..000000000 --- a/scrapy/newitem/loader/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -from collections import defaultdict - -from scrapy.utils.datatypes import MergeDict -from scrapy.newitem import Item -from scrapy.newitem.loader.reducers import TakeFirst -from scrapy.newitem.loader.expanders import IdentityExpander -from scrapy.xpath import HtmlXPathSelector - -class Loader(object): - - default_item_class = Item - default_expander = IdentityExpander() - default_reducer = TakeFirst() - - def __init__(self, item=None, **loader_args): - if item is None: - item = self.default_item_class() - self._item = loader_args['item'] = item - self._loader_args = loader_args - self._values = defaultdict(list) - - def add_value(self, field_name, value, **new_loader_args): - self._values[field_name] += self._expand_value(field_name, value, \ - new_loader_args) - - def replace_value(self, field_name, value, **new_loader_args): - self._values[field_name] = self._expand_value(field_name, value, \ - new_loader_args) - - def get_item(self): - item = self._item - for field_name in self._values: - item[field_name] = self.get_reduced_value(field_name) - return item - - def get_expanded_value(self, field_name): - return self._values[field_name] - - def get_reduced_value(self, field_name): - reducer = self.get_reducer(field_name) - return reducer(self._values[field_name]) - - def get_expander(self, field_name): - expander = getattr(self, '%s_exp' % field_name, None) - if not expander: - expander = self._item.fields[field_name].get('expander', \ - self.default_expander) - return expander - - def get_reducer(self, field_name): - reducer = getattr(self, '%s_red' % field_name, None) - if not reducer: - reducer = self._item.fields[field_name].get('reducer', \ - self.default_reducer) - return reducer - - def _expand_value(self, field_name, value, new_loader_args): - loader_args = self._loader_args - if new_loader_args: - loader_args = MergeDict(new_loader_args, self._loader_args) - expander = self.get_expander(field_name) - return expander(value, loader_args=loader_args) - - -class XPathLoader(Loader): - - default_selector_class = HtmlXPathSelector - - def __init__(self, item=None, selector=None, response=None, **loader_args): - if selector is None and response is None: - raise RuntimeError("%s must be instantiated with a selector" \ - "or response" % self.__class__.__name__) - if selector is None: - selector = self.default_selector_class(response) - self.selector = selector - loader_args.update(selector=selector, response=response) - super(XPathLoader, self).__init__(item, **loader_args) - - def add_xpath(self, field_name, xpath, re=None, **new_loader_args): - self.add_value(field_name, self._get_values(field_name, xpath, re), - **new_loader_args) - - def replace_xpath(self, field_name, xpath, re=None, **new_loader_args): - self.replace_value(field_name, self._get_values(field_name, xpath, re), \ - **new_loader_args) - - def _get_values(self, field_name, xpath, re): - x = self.selector.x(xpath) - return x.re(re) if re else x.extract() diff --git a/scrapy/newitem/loader/expanders.py b/scrapy/newitem/loader/expanders.py deleted file mode 100644 index 3e7d77e73..000000000 --- a/scrapy/newitem/loader/expanders.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This module provides some commonly used Expanders. - -See documentation in docs/topics/newitem-loader.rst -""" - -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args -from scrapy.utils.datatypes import MergeDict - -class TreeExpander(object): - - def __init__(self, *functions, **default_loader_args): - self.default_loader_args = default_loader_args - self.wrapped_funcs = [] - for func in functions: - if 'loader_args' in get_func_args(func): - wfunc = self.wrap_with_args(func) - else: - wfunc = self.wrap_no_args(func) - self.wrapped_funcs.append(wfunc) - - def wrap_no_args(self, f): - return lambda x, y: f(x) - - def wrap_with_args(self, f): - return lambda x, y: f(x, loader_args=y) - - def __call__(self, value, loader_args=None): - values = arg_to_iter(value) - largs = self.default_loader_args - if loader_args: - largs = MergeDict(loader_args, self.default_loader_args) - for func in self.wrapped_funcs: - next_values = [] - for v in values: - next_values += arg_to_iter(func(v, largs)) - values = next_values - return list(values) - - -class IdentityExpander(object): - - def __call__(self, values, loader_args): - return arg_to_iter(values) diff --git a/scrapy/newitem/loader/reducers.py b/scrapy/newitem/loader/reducers.py deleted file mode 100644 index 8f9a3cc9b..000000000 --- a/scrapy/newitem/loader/reducers.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -This module provides some commonly used Reducers. - -See documentation in docs/topics/newitem-loader.rst -""" - -class TakeFirst(object): - - def __call__(self, values): - for value in values: - if value: - return value - - -class Identity(object): - - def __call__(self, values): - return values - - -class Join(object): - - def __init__(self, separator=u' '): - self.separator = separator - - def __call__(self, values): - return self.separator.join(values) diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py deleted file mode 100644 index a32d2b036..000000000 --- a/scrapy/tests/test_itemloader.py +++ /dev/null @@ -1,272 +0,0 @@ -import unittest - -from scrapy.newitem.loader import Loader, XPathLoader -from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander -from scrapy.newitem.loader.reducers import Join, Identity -from scrapy.newitem import Item, Field -from scrapy.xpath import HtmlXPathSelector -from scrapy.http import HtmlResponse - -# test items - -class NameItem(Item): - name = Field() - -class TestItem(NameItem): - url = Field() - summary = Field() - -# test loaders - -class NameLoader(Loader): - default_item_class = TestItem - -class TestLoader(NameLoader): - name_exp = TreeExpander(lambda v: v.title()) - -class DefaultedLoader(NameLoader): - default_expander = TreeExpander(lambda v: v[:-1]) - -# test expanders - -def expander_func_with_args(value, other=None, loader_args=None): - if 'key' in loader_args: - return loader_args['key'] - return value - -class LoaderTest(unittest.TestCase): - - def test_get_item_using_default_loader(self): - i = TestItem() - i['summary'] = u'lala' - il = Loader(item=i) - il.add_value('name', u'marta') - item = il.get_item() - assert item is i - self.assertEqual(item['summary'], u'lala') - self.assertEqual(item['name'], u'marta') - - def test_get_item_using_custom_loader(self): - il = TestLoader() - il.add_value('name', u'marta') - item = il.get_item() - self.assertEqual(item['name'], u'Marta') - - def test_add_value(self): - il = TestLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_expanded_value('name'), [u'Marta']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - il.add_value('name', u'pepe') - self.assertEqual(il.get_expanded_value('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - def test_replace_value(self): - il = TestLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_expanded_value('name'), [u'Marta']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - il.replace_value('name', u'pepe') - self.assertEqual(il.get_expanded_value('name'), [u'Pepe']) - self.assertEqual(il.get_reduced_value('name'), u'Pepe') - - def test_tree_expander_filter(self): - def filter_world(x): - return None if x == 'world' else x - - expander = TreeExpander(filter_world, str.upper) - self.assertEqual(expander(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) - - def test_tree_expander_multiple_functions(self): - class TestLoader(NameLoader): - name_exp = TreeExpander(lambda v: v.title(), lambda v: v[:-1]) - - il = TestLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Mart') - item = il.get_item() - self.assertEqual(item['name'], u'Mart') - - def test_default_expander(self): - il = DefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mart') - - def test_inherited_default_expander(self): - class InheritDefaultedLoader(DefaultedLoader): - pass - - il = InheritDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mart') - - def test_expander_inheritance(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(lambda v: v.lower()) - - il = ChildLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org') - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - class ChildChildLoader(ChildLoader): - url_exp = TreeExpander(lambda v: v.upper()) - summary_exp = TreeExpander(lambda v: v) - - il = ChildChildLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG') - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - def test_empty_tree_expander(self): - class IdentityDefaultedLoader(DefaultedLoader): - name_exp = TreeExpander() - - il = IdentityDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'marta') - - def test_identity_expander(self): - class IdentityDefaultedLoader(DefaultedLoader): - name_exp = IdentityExpander() - - il = IdentityDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'marta') - - def test_extend_custom_expanders(self): - class ChildLoader(TestLoader): - name_exp = TreeExpander(TestLoader.name_exp, unicode.swapcase) - - il = ChildLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mARTA') - - def test_extend_default_expanders(self): - class ChildDefaultedLoader(DefaultedLoader): - name_exp = TreeExpander(DefaultedLoader.default_expander, unicode.swapcase) - - il = ChildDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'MART') - - def test_reducer_using_function(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class TakeFirstLoader(TestLoader): - name_red = u" ".join - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - - def test_reducer_using_classes(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class TakeFirstLoader(TestLoader): - name_red = Join() - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - - class TakeFirstLoader(TestLoader): - name_red = Join("
") - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar
Ta') - - def test_default_reducer(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class LalaLoader(TestLoader): - default_reducer = Identity() - - il = LalaLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), [u'Mar', u'Ta']) - - def test_expander_args_on_declaration(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args, key=u'val') - - il = ChildLoader() - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_expander_args_on_instantiation(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args) - - il = ChildLoader(key=u'val') - il.add_value('url', u'text') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_expander_args_on_assign(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args) - - il = ChildLoader() - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_item_passed_to_expander_functions(self): - def exp_func(value, loader_args): - return loader_args['item']['name'] - - class ChildLoader(TestLoader): - url_exp = TreeExpander(exp_func) - - it = TestItem(name='marta') - il = ChildLoader(item=it) - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'marta') - - def test_add_value_on_unknown_field(self): - il = TestLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) - - -class TestXPathLoader(XPathLoader): - default_item_class = TestItem - name_exp = TreeExpander(lambda v: v.title()) - -class XPathLoaderTest(unittest.TestCase): - - def test_constructor_errors(self): - self.assertRaises(RuntimeError, XPathLoader) - - def test_constructor_with_selector(self): - sel = HtmlXPathSelector(text=u"
marta
") - l = TestXPathLoader(selector=sel) - self.assert_(l.selector is sel) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_reduced_value('name'), u'Marta') - - def test_constructor_with_response(self): - response = HtmlResponse(url="", body="
marta
") - l = TestXPathLoader(response=response) - self.assert_(l.selector) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_reduced_value('name'), u'Marta') - - def test_add_xpath_re(self): - response = HtmlResponse(url="", body="
marta
") - l = TestXPathLoader(response=response) - l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_reduced_value('name'), u'Ma') - - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/tests/test_itemparser.py b/scrapy/tests/test_itemparser.py new file mode 100644 index 000000000..cf0e8addc --- /dev/null +++ b/scrapy/tests/test_itemparser.py @@ -0,0 +1,280 @@ +import unittest + +from scrapy.contrib.itemparser import ItemParser, XPathItemParser +from scrapy.contrib.itemparser.parsers import ApplyConcat, Join, Identity +from scrapy.newitem import Item, Field +from scrapy.xpath import HtmlXPathSelector +from scrapy.http import HtmlResponse + +# test items + +class NameItem(Item): + name = Field() + +class TestItem(NameItem): + url = Field() + summary = Field() + +# test item parsers + +class NameItemParser(ItemParser): + default_item_class = TestItem + +class TestItemParser(NameItemParser): + name_in = ApplyConcat(lambda v: v.title()) + +class DefaultedItemParser(NameItemParser): + default_input_parser = ApplyConcat(lambda v: v[:-1]) + +# test parsers + +def parser_with_args(value, other=None, parser_context=None): + if 'key' in parser_context: + return parser_context['key'] + return value + +class ItemParserTest(unittest.TestCase): + + def test_populate_item_using_default_loader(self): + i = TestItem() + i['summary'] = u'lala' + ip = ItemParser(item=i) + ip.add_value('name', u'marta') + item = ip.populate_item() + assert item is i + self.assertEqual(item['summary'], u'lala') + self.assertEqual(item['name'], [u'marta']) + + def test_populate_item_using_custom_loader(self): + ip = TestItemParser() + ip.add_value('name', u'marta') + item = ip.populate_item() + self.assertEqual(item['name'], [u'Marta']) + + def test_add_value(self): + ip = TestItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_collected_values('name'), [u'Marta']) + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + ip.add_value('name', u'pepe') + self.assertEqual(ip.get_collected_values('name'), [u'Marta', u'Pepe']) + self.assertEqual(ip.get_output_value('name'), [u'Marta', u'Pepe']) + + def test_replace_value(self): + ip = TestItemParser() + ip.replace_value('name', u'marta') + self.assertEqual(ip.get_collected_values('name'), [u'Marta']) + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + ip.replace_value('name', u'pepe') + self.assertEqual(ip.get_collected_values('name'), [u'Pepe']) + self.assertEqual(ip.get_output_value('name'), [u'Pepe']) + + def test_map_concat_filter(self): + def filter_world(x): + return None if x == 'world' else x + + parser = ApplyConcat(filter_world, str.upper) + self.assertEqual(parser(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) + + def test_map_concat_filter_multiple_functions(self): + class TestItemParser(NameItemParser): + name_in = ApplyConcat(lambda v: v.title(), lambda v: v[:-1]) + + ip = TestItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Mart']) + item = ip.populate_item() + self.assertEqual(item['name'], [u'Mart']) + + def test_default_input_parser(self): + ip = DefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mart']) + + def test_inherited_default_input_parser(self): + class InheritDefaultedItemParser(DefaultedItemParser): + pass + + ip = InheritDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mart']) + + def test_input_parser_inheritance(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(lambda v: v.lower()) + + ip = ChildItemParser() + ip.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(ip.get_output_value('url'), [u'http://scrapy.org']) + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + + class ChildChildItemParser(ChildItemParser): + url_in = ApplyConcat(lambda v: v.upper()) + summary_in = ApplyConcat(lambda v: v) + + ip = ChildChildItemParser() + ip.add_value('url', u'http://scrapy.org') + self.assertEqual(ip.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + + def test_empty_map_concat(self): + class IdentityDefaultedItemParser(DefaultedItemParser): + name_in = ApplyConcat() + + ip = IdentityDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'marta']) + + def test_identity_input_parser(self): + class IdentityDefaultedItemParser(DefaultedItemParser): + name_in = Identity() + + ip = IdentityDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'marta']) + + def test_extend_custom_input_parsers(self): + class ChildItemParser(TestItemParser): + name_in = ApplyConcat(TestItemParser.name_in, unicode.swapcase) + + ip = ChildItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mARTA']) + + def test_extend_default_input_parsers(self): + class ChildDefaultedItemParser(DefaultedItemParser): + name_in = ApplyConcat(DefaultedItemParser.default_input_parser, unicode.swapcase) + + ip = ChildDefaultedItemParser() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'MART']) + + def test_output_parser_using_function(self): + ip = TestItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemParser(TestItemParser): + name_out = u" ".join + + ip = TakeFirstItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar Ta') + + def test_output_parser_using_classes(self): + ip = TestItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemParser(TestItemParser): + name_out = Join() + + ip = TakeFirstItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar Ta') + + class TakeFirstItemParser(TestItemParser): + name_out = Join("
") + + ip = TakeFirstItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar
Ta') + + def test_default_output_parser(self): + ip = TestItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class LalaItemParser(TestItemParser): + default_output_parser = Identity() + + ip = LalaItemParser() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + def test_parser_context_on_declaration(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser_with_args, key=u'val') + + ip = ChildItemParser() + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_parser_context_on_instantiation(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser_with_args) + + ip = ChildItemParser(key=u'val') + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_parser_context_on_assign(self): + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser_with_args) + + ip = ChildItemParser() + ip.context['key'] = u'val' + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_item_passed_to_input_parser_functions(self): + def parser(value, parser_context): + return parser_context['item']['name'] + + class ChildItemParser(TestItemParser): + url_in = ApplyConcat(parser) + + it = TestItem(name='marta') + ip = ChildItemParser(item=it) + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['marta']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['marta']) + + def test_add_value_on_unknown_field(self): + ip = TestItemParser() + self.assertRaises(KeyError, ip.add_value, 'wrong_field', [u'lala', u'lolo']) + + +class TestXPathItemParser(XPathItemParser): + default_item_class = TestItem + name_in = ApplyConcat(lambda v: v.title()) + +class XPathItemParserTest(unittest.TestCase): + + def test_constructor_errors(self): + self.assertRaises(RuntimeError, XPathItemParser) + + def test_constructor_with_selector(self): + sel = HtmlXPathSelector(text=u"
marta
") + l = TestXPathItemParser(selector=sel) + self.assert_(l.selector is sel) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'Marta']) + + def test_constructor_with_response(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathItemParser(response=response) + self.assert_(l.selector) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'Marta']) + + def test_add_xpath_re(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathItemParser(response=response) + l.add_xpath('name', '//div/text()', re='ma') + self.assertEqual(l.get_output_value('name'), [u'Ma']) + + +if __name__ == "__main__": + unittest.main() +