diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 55468737d..d7eaf261c 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -20,4 +20,4 @@ it's properly merged) . Use at your own risk. :maxdepth: 1 newitems - loaders + itemparser diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst index dfec482d2..d96defb71 100644 --- a/docs/experimental/loaders.rst +++ b/docs/experimental/loaders.rst @@ -1,55 +1,55 @@ -.. _topics-loader: +.. _topics-loaders: ============ Item Loaders ============ -.. module:: scrapy.newitem.loader +.. module:: scrapy.contrib.loader :synopsis: Item Loader class -Item Loaders (or Loaders, for short) provide a convenient mechanism for -populating scraped :ref:`Items `. Even though Items can be -populated using their own dictionary-like API, the Loaders provide a much more -convenient API for populating them from a scraping process, by automating some -common tasks like parsing the raw extracted data before assigning it. +Item Loaders provide a convenient mechanism for populating scraped :ref:`Items +`. Even though Items can be populated using their own +dictionary-like API, the Item Loaders provide a much more convenient API for +populating them from a scraping process, by automating some common tasks like +parsing the raw extracted data before assigning it. In other words, :ref:`Items ` provide the *container* of -scraped data, while Loaders provide the mechanism for *populating* that +scraped data, while Item Loaders provide the mechanism for *populating* that container. -Loaders are designed to provide a flexible, efficient and easy mechanism for -extending and overriding different field parsing rules (either by spider, or by -source format) without becoming a nightmare to maintain +Item Loaders are designed to provide a flexible, efficient and easy mechanism +for extending and overriding different field parsing rules, either by spider, +or by source format (HTML, XML, etc) without becoming a nightmare to maintain. -Using Loaders to populate items -=============================== +Using Item Loaders to populate items +==================================== -To use a Loader, you must first instantiate it. You can either instantiate it -with an Item object or without one, in which case an Item is automatically -instantiated in the Loader constructor using the Item class specified in the -:attr:`Loader.default_item_class` attribute. +To use an Item Loader, you must first instantiate it. You can either +instantiate it with an Item object or without one, in which case an Item is +automatically instantiated in the Item Loader constructor using the Item class +specified in the :attr:`ItemLoader.default_item_class` attribute. -Then, you start adding values to the Loader, typically collecting them using -:ref:`Selectors `. You can add more than one value to the -same item field, the Loader will know how to "join" those values later using a -Reducer. +Then, you start collecting values into the Item Loader, typically using using +:ref:`XPath Selectors `. You can add more than one value to +the same item field, the Item Loader will know how to "join" those values later +using a proper processing function. -Here is a typical Loader usage in a :ref:`Spider `, using the -:ref:`Product item ` declared in the :ref:`Items -section `:: +Here is a typical Item Loader usage in a :ref:`Spider `, using +the :ref:`Product item ` declared in the :ref:`Items +chapter `:: - from scrapy.item.loader import XPathLoader + from scrapy.contrib.loader import XPathItemLoader from scrapy.xpath import HtmlXPathSelector from myproject.items import Product def parse(self, response): - l = XPathLoader(item=Product(), response=response) - l.add_xpath('name', '//div[@class="product_name"]') - l.add_xpath('name', '//div[@class="product_title"]') - l.add_xpath('price', '//p[@id="price"]') - l.add_xpath('stock', '//p[@id="stock"]') - l.add_value('last_updated', 'today') # you can also use literal values - return l.get_item() + p = XPathItemLoader(item=Product(), response=response) + p.add_xpath('name', '//div[@class="product_name"]') + p.add_xpath('name', '//div[@class="product_title"]') + p.add_xpath('price', '//p[@id="price"]') + p.add_xpath('stock', '//p[@id="stock"]') + p.add_value('last_updated', 'today') # you can also use literal values + return p.populate_item() By quickly looking at that code we can see the ``name`` field is being extracted from two different XPath locations in the page: @@ -58,178 +58,199 @@ extracted from two different XPath locations in the page: 2. ``//div[@class="product_title"]`` In other words, data is being collected by extracting it from two XPath -locations, using the :meth:`~XPathLoader.add_xpath` method. This is the data +locations, using the :meth:`~XPathItemLoader.add_xpath` method. This is the data that will be assigned to the ``name`` field later. Afterwards, similar calls are used for ``price`` and ``stock`` fields, and finally the ``last_update`` field is populated directly with a literal value -(``today``) using a different method: :meth:`~Loader.add_value`. +(``today``) using a different method: :meth:`~ItemLoader.add_value`. -Finally, when all data is collected, the :meth:`Loader.get_item` method is -called which actually populates and returns the item populated with the data -previously extracted and collected with the :meth:`~XPathLoader.add_xpath` and -:meth:`~Loader.add_value` calls. +Finally, when all data is collected, the :meth:`ItemLoader.populate_item` +method is called which actually populates and returns the item populated with +the data previously extracted and collected with the +:meth:`~XPathItemLoader.add_xpath` and :meth:`~ItemLoader.add_value` calls. -.. _topics-loader-expred: +.. _topics-loaders-processors: -Expanders and Reducers -====================== +Input and Output processors +=========================== -A Loader is composed of one expander and one reducer for each (item) field. The -Expander processes the extracted data as soon as it's received (through the -:meth:`~XPathLoader.add_xpath` or :meth:`~Loader.add_value` methods) and the -result of the expander is collected and kept inside the Loader. After -collecting all data, the :meth:`Loader.get_item` method is called to actually -populate and get the :class:`~scrapy.newitem.Item` object. That's when the -Reducers are called with the data previously collected (using the Expanders) -and the output of the Reducers are the actual values that get assigned to the -item. +An Item Loader contains one input processor and one output processor for each +(item) field. The input processor processes the extracted data as soon as it's +received (through the :meth:`~XPathItemLoader.add_xpath` or +:meth:`~ItemLoader.add_value` methods) and the result of the input processor is +collected and kept inside the ItemLoader. After collecting all data, the +:meth:`ItemLoader.populate_item` method is called to populate and get the +populated :class:`~scrapy.newitem.Item` object. That's when the output processor +is called with the data previously collected (and processed using the input +processor). The result of the output processor is the final value that gets assigned +to the item. -Let's see an example to illustrate how Expanders and Reducers are called, for a -particular field (the same applies for any other field):: +Let's see an example to illustrate how this input and output processors are +called for a particular field (the same applies for any other field):: - l = XPathLoader(Product(), some_selector) - l.add_xpath('name', xpath1) # (1) - l.add_xpath('name', xpath2) # (2) - return l.get_item() # (3) + p = XPathItemLoader(Product(), some_xpath_selector) + p.add_xpath('name', xpath1) # (1) + p.add_xpath('name', xpath2) # (2) + return p.populate_item() # (3) So what happens is: -1. Data from ``xpath1`` is extracted, and passed through the Expander of the - ``name`` field. The output of the expander is collected and kept in the - loader (but not yet assigned to the item). +1. Data from ``xpath1`` is extracted, and passed through the *input processor* of + the ``name`` field. The result of the input processor is collected and kept in + the Item Loader (but not yet assigned to the item). -2. Data from ``xpath2`` is extracted, and passed through the same Expander used - in (1). The output of the expander is appended to the data collected in (1) - (if any). +2. Data from ``xpath2`` is extracted, and passed through the same *input + processor* used in (1). The result of the input processor is appended to the + data collected in (1) (if any). -3. The data collected in (1) and (2) is passed through the Reducer of the - ``name`` field. The output of the Reducer is the value assigned to the - ``name`` field in the item. +3. The data collected in (1) and (2) is passed through the *output processor* of + the ``name`` field. The result of the output processor is the value assigned to + the ``name`` field in the item. -Scrapy comes with one major expander built-in, the :ref:`Tree Expander -`, and :ref:`a couple of commonly used reducers -`. +It's worth noticing that processors are just callable objects, which are called +with the data to be parsed, and return a parsed value. So you can use any +function as input or output processor, provided they can receive only one +positional (required) argument. -Declaring Loaders -================= +The other thing you need to keep in mind is that the values returned by input +processors are collected internally (in lists) and then passed to output +processors to populate the fields, so output processors should expect iterables as +input. -Loaders are declared like Items, by using a class definition syntax. Here is an -example:: +Last, but not least, Scrapy comes with some :ref:`commonly used processors +` built-in for convenience. - from scrapy.newitem.loader import Loader - from scrapy.newitem.loader.expanders import TreeExpander - from scrapy.newitem.loader.reducers import Join, TakeFirst - class ProductLoader(Loader): +Declaring Item Loaders +====================== - default_expander = TakeFirst() +Item Loaders are declared like Items, by using a class definition syntax. Here +is an example:: - name_exp = TreeExpander(unicode.title) - name_red = Join() + from scrapy.contrib.loader import ItemLoader + from scrapy.contrib.loader.processor import TakeFirst, ApplyConcat, Join - price_exp = TreeExpander(unicode.strip) - price_red = TakeFirst() + class ProductLoader(ItemLoader): - ... + default_input_processor = TakeFirst() -As you can see, expanders are declared using the ``_exp`` suffix while reducers -are declared using the ``_red`` suffix. And you can also declare a default -expander using the :attr:`Loader.default_expander` attribute. + name_in = ApplyConcat(unicode.title) + name_out = Join() -.. _topics-loader-expred-declaring: + price_in = ApplyConcat(unicode.strip) + price_out = TakeFirst() -Declaring Extenders and Reducers -================================ + # ... -As seen in the previous section, extenders and reducers can be declared in the -Loader definition, and it's very common to declare expanders this way. However, -there is one more place where you can specify the exanders and reducers to use: -in the :ref:`Item Field ` metadata. Here is an example:: +As you can see, input processors are declared using the ``_in`` suffix while +output processors are declared using the ``_out`` suffix. And you can also +declare a default input/output processors using the +:attr:`ItemLoader.default_input_processor` and +:attr:`ItemLoader.default_output_processor` attributes. + +.. _topics-loaders-processors-declaring: + +Declaring Input and Output Processors +===================================== + +As seen in the previous section, input and output processors can be declared in +the Item Loader definition, and it's very common to declare input processors +this way. However, there is one more place where you can specify the input and +output processors to use: in the :ref:`Item Field ` +metadata. Here is an example:: from scrapy.newitem import Item, Field - from scrapy.newitem.loader.expanders import TreeExpander - from scrapy.newitem.loader.reducers import Join, TakeFirst + from scrapy.contrib.loader.processor import ApplyConcat, Join, TakeFirst from scrapy.utils.markup import remove_entities - from myprojct.utils import filter_prices + from myproject.utils import filter_prices class Product(Item): name = Field( - expander=TreeExpander(remove_entities), - reducer=Join(), + input_processor=ApplyConcat(remove_entities), + output_processor=Join(), ) price = Field( default=0, - expander=TreeExpander(remove_entities, filter_prices), - reducer=TakeFirst(), + input_processor=ApplyConcat(remove_entities, filter_prices), + output_processor=TakeFirst(), ) -The precendece order, for both expander and reducer declarations, is as -follows: +The precedence order, for both input and output processors, is as follows: -1. Loader field-specific attributes: ``field_exp`` and ``field_red`` (more +1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most precedence) -2. Field metadata (``expander`` and ``reducer`` key) -3. Loader defaults: :meth:`Loader.default_expander` and - :meth:`Loader.default_reducer` (less precedence) +2. Field metadata (``input_processor`` and ``output_processor`` key) +3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and + :meth:`ItemLoader.default_output_processor` (least precedence) -See also: :ref:`topics-loader-extending`. +See also: :ref:`topics-loaders-extending`. -.. _topics-loader-args: +.. _topics-loaders-context: -Item Loader arguments -===================== +Item Loader Context +=================== -The Loader arguments is a dict of arbitrary key/values which can be passed when -declaring, instantiating or using Loaders. They are used modify the behaviour -of the expanders. +The Item Loader Context is a dict of arbitrary key/values which is shared among +all input and output processors in the Item Loader. It can be passed when +declaring, instantiating or using Item Loader. They are used to modify the +behaviour of the input/output processors. For example, suppose you have a function ``parse_length`` which receives a text value and extracts a length from it:: - def parse_length(text, loader_args): - unit = loader_args('unit', 'm') + def parse_length(text, loader_context): + unit = loader_context.get('unit', 'm') # ... length parsing code goes here ... return parsed_length -Since it receives a ``loader_args`` the Expander will pass the currently active -Loader arguments when calling it. +By accepting a ``loader_context`` argument the function is explicitly telling +the Item Loader that is able to receive an Item Loader context, so the Item +Loader passes the currently active context when calling it, and the processor +function (``parse_length`` in this case) can thus use them. -There are seveal ways to pass Loader arguments: +There are several ways to modify Item Loader context values: -1. Passing arguments on Loader declaration:: +1. By modifying the currently active Item Loader context +(:meth:`ItemLoader.context` attribute):: - class ProductLoader(Loader): - length_exp = TreeExpander(parse_length, unit='cm') + loader = ItemLoader(product, unit='cm') + loader.context['unit'] = 'cm' -2. Passing arguments on Loader instantiation:: +2. On Item Loader instantiation (the keyword arguments of Item Loader + constructor are stored in the Item Loader context):: - l = Loader(product, unit='cm') + p = ItemLoader(product, unit='cm') -3. Passing arguments on Loader usage:: +2. On Item Loader declaration, for those input/output processors that support + instatiating them with a Item Loader context. :class:`ApplyConcat` is one of + them:: - l.add_xpath('length', '//div', unit='cm') + class ProductLoader(ItemLoader): + length_out = ApplyConcat(parse_length, unit='cm') -Loader objects -============== -.. class:: Loader([item], \**loader_args) +ItemLoader objects +================== + +.. class:: ItemLoader([item], \**kwargs) Return a new Item Loader for populating the given Item. If no item is - given, one is instantiated using the class in :attr:`default_item_class`. + given, one is instantiated automatically using the class in + :attr:`default_item_class`. - .. method:: add_value(field_name, value, \**new_loader_args) + The item and the remaining keyword arguments are assigned to the Loader + context (accesible through the :attr:`context` attribute). + + .. method:: add_value(field_name, value) Add the given ``value`` for the given field. - The value is passed through the :ref:`field expander - ` and its output appened to the data collected - for that field. If the field already contains collected data, the new - data is added. - - If any keyword arguments are passed, they're used as :ref:`Loader - arguments ` when calling the expanders. + The value is passed through the :ref:`field input processor + ` and its result appened to the data + collected for that field. If the field already contains collected data, + the new data is added. Examples:: @@ -237,60 +258,68 @@ Loader objects loader.add_value('colours', [u'white', u'blue']) loader.add_value('length', u'100', default_unit='cm') - .. method:: replace_value(field_name, value, \**new_loader_args) + .. method:: replace_value(field_name, value) - Similar to :meth:`add_value` but replaces collected data instead of - adding it. + Similar to :meth:`add_value` but replaces the collected data with the + new value instead of adding it. - - .. method:: get_item() + .. method:: populate_item() Populate the item with the data collected so far, and return it. The - data collected is first passed through the :ref:`field reducers - ` to get the final value to assign to each item - field. + data collected is first passed through the :ref:`field output processors + ` to get the final value to assign to each + item field. - .. method:: get_expanded_value(field_name) + .. method:: get_collected_values(field_name) - Return the expanded data for the given field. In other words, return - the dat collected so far for the given field, without reducing it. + Return the collected values for the given field. - .. method:: get_reduced_value(field_name) + .. method:: get_output_value(field_name) - Return the reduced value for the given field, without modifying the - item. + Return the collected values parsed using the output processor, for the + given field. This method doesn't populate or modify the item at all. - .. method:: get_expander(field_name) + .. method:: get_input_processor(field_name) - Return the expander for the given field. + Return the input processor for the given field. - .. method:: get_reducer(field_name) + .. method:: get_output_processor(field_name) - Return the reducer for the given field. + Return the output processor for the given field. + + .. attribute:: item + + The :class:`~scrapy.newitem.Item` object being parsed by this Item + Loader. + + .. attribute:: context + + The currently active :ref:`Context ` of this + Item Loader. .. attribute:: default_item_class An Item class (or factory), used to instantiate items when not given in the constructor. - .. attribute:: default_expander + .. attribute:: default_input_processor - The default expander to use for those fields which don't define a - specific expander + The default input processor to use for those fields which don't specify + one. - .. attribute:: default_reducer + .. attribute:: default_output_processor - The default reducer to use for those fields which don't define a - specific expander + The default output processor to use for those fields which don't specify + one. -.. class:: XPathLoader([item, selector, response], \**loader_args) +.. class:: XPathItemLoader([item, selector, response], \**kwargs) - The :class:`XPathLoader` class extends the :class:`Loader` class providing - more convenient mechanisms for extracting data from web pages using - :ref:`XPath selectors `. + The :class:`XPathItemLoader` class extends the :class:`ItemLoader` class + providing more convenient mechanisms for extracting data from web pages + using :ref:`XPath selectors `. - :class:`XPathLoader` objects accept two more additional parameters in their - constructors: + :class:`XPathItemLoader` objects accept two more additional parameters in + their constructors: :param selector: The selector to extract data from, when using the :meth:`add_xpath` or :meth:`replace_xpath` method. @@ -301,11 +330,11 @@ Loader objects in which case this argument is ignored. :type response: :class:`~scrapy.http.Response` object - .. method:: add_xpath(field_name, xpath, re=None, \**new_loader_args) + .. method:: add_xpath(field_name, xpath, re=None) - Similar to :meth:`Loader.add_value` but receives an XPath instead of a + Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a value, which is used to extract a list of unicode strings from the - selector associated with this :class:`XPathLoader`. If the ``re`` + selector associated with this :class:`XPathItemLoader`. If the ``re`` argument is given, it's used for extrating data from the selector using the :meth:`~scrapy.xpath.XPathSelector.re` method. @@ -323,7 +352,7 @@ Loader objects # HTML snippet:

the price is $1200

loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - .. method:: replace_xpath(field_name, xpath, re=None, \**new_loader_args) + .. method:: replace_xpath(field_name, xpath, re=None) Similar to :meth:`add_xpath` but replaces collected data instead of adding it. @@ -331,7 +360,7 @@ Loader objects .. attribute:: default_selector_class The class used to construct the :attr:`selector` of this - :class:`XPathLoader`, if only a response is given in the constructor. + :class:`XPathItemLoader`, if only a response is given in the constructor. If a selector is given in the constructor this attribute is ignored. This attribute is sometimes overridden in subclasses. @@ -343,159 +372,156 @@ Loader objects :attr:`default_selector_class`. This attribute is meant to be read-only. -.. _topics-loader-extending: +.. _topics-loaders-extending: -Reusing and extending Loaders -============================= +Reusing and extending Item Loaders +================================== As your project grows bigger and acquires more and more spiders, maintenance becomes a fundamental problem, specially when you have to deal with many -different parsing rules per spider, a lot of exceptions, but also want to reuse -the common cases. +different parsing rules for each spider, having a lot of exceptions, but also +wanting to reuse the common processors. -Loaders are designed to ease the maintenance of parsing rules, without loosing -flexibility and, at the same time, providing a convenient mechanism for -extending and overriding them. For this reason Loaders support traditional -class inheritance for for dealing with differences of specific spiders (or -group of spiders). +Item Loaders are designed to ease the maintenance burden of parsing rules, +without loosing flexibility and, at the same time, providing a convenient +mechanism for extending and overriding them. For this reason Item Loaders +support traditional Python class inheritance for dealing with differences of +specific spiders (or group of spiders). -Suppose, for example, that some particular site encloses their product names -between three dashes (ie. ``---Plasma TV---``) and you don't want to end up -scraping those dashes in the final product names. +Suppose, for example, that some particular site encloses their product names in +three dashes (ie. ``---Plasma TV---``) and you don't want to end up scraping +those dashes in the final product names. Here's how you can remove those dashes by reusing and extending the default -Product Loader:: +Product Item Loader (``ProductLoader``):: - strip_dashes = lambda x: x.strip('-') + from scrapy.contrib.loader.processor import ApplyConcat + from myproject.ItemLoaders import ProductLoader + + def strip_dashes(x): + return x.strip('-') class SiteSpecificLoader(ProductLoader): - name_exp = TreeExpander(ProductLoader.name_exp, strip_dashes) + name_in = ApplyConcat(ProductLoader.name_in, strip_dashes) -Another case where extending Loaders can be very helpful is when you have +Another case where extending Item Loaders can be very helpful is when you have multiple source formats, for example XML and HTML. In the XML version you may want to remove ``CDATA`` occurrences. Here's an example of how to do it:: + from scrapy.contrib.loader.processor import ApplyConcat + from myproject.ItemLoaders import ProductLoader from myproject.utils.xml import remove_cdata - class XmlLoader(ProductLoader): - name_exp = TreeExpander(remove_cdata, ProductLoader.name_exp) + class XmlProductLoader(ProductLoader): + name_in = ApplyConcat(remove_cdata, ProductLoader.name_in) -And that's how you typically extend expanders. +And that's how you typically extend input processors. -As for reducers, it is more common to declare them in the field metadata, as -they usually depend only on the field and not on each specific site parsing -rule (as expanders do). See also: :ref:`topics-loader-expred-declaring`. +As for output processors, it is more common to declare them in the field metadata, +as they usually depend only on the field and not on each specific site parsing +rule (as input processors do). See also: +:ref:`topics-loaders-processors-declaring`. -There are many other possible ways to extend, inherit and override your -Loaders, and different Loader hierarchies may fit better for different +There are many other possible ways to extend, inherit and override your Item +Loaders, and different Item Loaders hierarchies may fit better for different projects. Scrapy only provides the mechanism, it doesn't impose any specific organization of your Loaders collection - that's up to you and your project needs. -Available Expanders -=================== +.. _topics-loaders-available-processors: -.. _topics-loader-tree-expander: +Available built-in processors +============================= -Tree Expander -------------- +Even though you can use any callable function as input and output processors, +Scrapy provides some commonly used processors, which are described below. Some +of them, like the :class:`ApplyConcat` (which is typically used as input +processor) composes the output of several functions executed in order, to +produce the final parsed value. -The Tree Expander is the recommended Expander to use and the only really useful -one, as the other is just an identity expander. +Here is a list of all built-in processors: -.. module:: scrapy.newitem.loader.expanders - :synopsis: Expander classes to use with Item Loaders +.. _topics-loaders-applyconcat: -.. class:: TreeExpander(\*functions, \**default_loader_arguments) +ApplyConcat processor +--------------------- - An expander which applies the given functions consecutively, in order, to - each value returned by the previous function. +The ApplyConcat processor is the recommended processor to use if you want to +concatenate the processing of several functions in a pipeline. - The algorithm consists in an ordered list of functions, each of which - receives one value and can return zero, one or more values (as a list or - iterable). If a function returns more than one value, the next function in - the list will be called with each of those values, potentially returning - more values and thus expanding the execution into different branches, which - is why this expander is called Tree Expander. +.. module:: scrapy.contrib.loader.processor + :synopsis: A collection of processors to use with Item Loaders - Each expander function can optionally receive a ``loader_args`` argument, - which will contain the currently active :ref:`Loader arguments - `. +.. class:: ApplyConcat(\*functions, \**default_loader_context) + + A processor which applies the given functions consecutively, in order, + concatenating their results before next function call. So each function + returns a list of values (though it could return ``None`` or a signle value + too) and the next function is called once for each of those values, + receiving one of those values as input each time. The output of each + function call (for each input value) is concatenated and each values of the + concatenation is used to call the next function, and the process repeats + until there are no functions left. + + Each function can optionally receive a ``loader_context`` parameter, which + will contain the currently active :ref:`Item Loader context + `. The keyword arguments passed in the consturctor are used as the default - Loader arguments passed to on each expander call. This arguments can be - overriden with specific noader arguments passed on each expander call. + Item Loader context values passed on each function call. However, the final + Item Loader context values passed to funtions get overriden with the + currently active Item Loader context accesible through the + :meth:`ItemLoader.context` attribute. Example:: >>> def filter_world(x): ... return None if x == 'world' else x ... - >>> from scrapy.newitem.loader.expanders import TreeExpander - >>> expander = TreeExpander(filter_world, str.upper) - >>> expander(['hello', 'world', 'this', 'is', 'scrapy']) + >>> from scrapy.contrib.loader.processor import ApplyConcat + >>> proc = ApplyConcat(filter_world, str.upper) + >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) ['HELLO, 'THIS', 'IS', 'SCRAPY'] - -IdentityExpander ----------------- - -.. class:: IdentityExpander - - An expander which returns the original values unchanged. It doesn't support - any constructor arguments. - -.. _topics-loader-reducers: - -Available Reducers -================== - -.. module:: scrapy.newitem.loader.reducers - :synopsis: Reducer classes to use with Item Loaders - -Reducers are callable objects which are called with a list of values (to be -reduced) as their first and only argument. Scrapy provides some simple, -commonly used reducers, which are described below. But you can use any function -or callable as reducer. - .. class:: TakeFirst - Return the first non-null value from the values to reduce, so it's used for - single-valued fields. It doesn't receive any constructor arguments. + Return the first non null/empty value from the values to received, so it's + typically used as output processor of single-valued fields. It doesn't + receive any constructor arguments, nor accepts a Item Loader context. Example:: - >>> from scrapy.newitem.loader.reducers import TakeFirst - >>> reducer = TakeFirst() - >>> reducer(['', 'one', 'two', 'three']) + >>> from scrapy.contrib.loader.processor import TakeFirst + >>> proc = TakeFirst() + >>> proc(['', 'one', 'two', 'three']) 'one' .. class:: Identity - Return the values to reduce unchanged, so it's used for multi-valued - fields. It doesn't receive any constructor arguments. + Return the original values unchanged. It doesn't receive any constructor + arguments nor accepts a Item Loader context. Example:: - >>> from scrapy.newitem.loader.reducers import Identity - >>> reducer = Identity() - >>> reducer(['one', 'two', 'three']) + >>> from scrapy.contrib.loader.processor import Identity + >>> proc = Identity() + >>> proc(['one', 'two', 'three']) ['one', 'two', 'three'] .. class:: Join(separator=u' ') - Return a the values to reduce joined with the separator given in the - constructor, which defaults to ``u' '``. + Return the values joined with the separator given in the constructor, which + defaults to ``u' '``. It doesn't accept a Item Loader context. - When using the default separator, this reducer is equivalent to the + When using the default separator, this processor is equivalent to the function: ``u' '.join`` Examples:: - >>> from scrapy.newitem.loader.reducers import Join - >>> reducer = Join() - >>> reducer(['one', 'two', 'three']) + >>> from scrapy.contrib.loader.processor import Join + >>> proc = Join() + >>> proc(['one', 'two', 'three']) u'one two three' - >>> reducer = Join('
') - >>> reducer(['one', 'two', 'three']) + >>> proc = Join('
') + >>> proc(['one', 'two', 'three']) u'one
two
three' diff --git a/docs/topics/adaptors.rst b/docs/topics/adaptors.rst deleted file mode 100644 index 9088beb62..000000000 --- a/docs/topics/adaptors.rst +++ /dev/null @@ -1,205 +0,0 @@ -.. _topics-adaptors: - -============================== -Old Item Adaptors (deprecated) -============================== - -.. warning:: - - This documentation is deprecated, and will be superseeded by the new item - adaptors (not yet documented). - -Quick overview -============== - -Scrapy's adaptors are a nice feature attached to :class:`RobustScrapedItem` -that allow you to easily modify (adapt to your needs) any kind of information -you want to put in your items at assignation time. - -The following diagram shows the data flow from the moment you call the -``attribute`` method until the attribute is actually set. - -.. image:: _images/adaptors_diagram.png - -As you can see, adaptor pipelines are executed in tree form; which means that, -for each of the values you pass to the ``attribute`` method, the first adaptor -will be applied. Then, for each of the resulting values of the first adaptor, -the second adaptor will be called, and so on. This process will end up with a -list of adapted values, which may contain zero, one, or many values. - -In case the attribute is a single-valued (this is defined in the item's -``ATTRIBUTES`` dictionary), the first element of this list will be set, unless -you call the ``attribute`` method with the add parameter as True, in which case -the item's method ``_add_single_attributes`` will be called with the -attribute's name, type, and the list of attributes to join as parameters. By -default, this method raises NotImplementedError, so you should override it in -your items in order to join any kind of objects. - -If the attribute is a multivalued, the resulting list will be set to the item -as is, unless you use -again- add=True, in which case the list of -already-existing values (if any) will be extended with the new one.pgq - -Adaptor Pipelines -================= - -.. class:: AdaptorPipe(adaptors=None) - - An instance of this class represents an adaptor pipeline to be set for - adapting a certain item's attribute. It provides some useful methods for - adding/removing adaptors, and takes care of executing them properly. - Usually this class is not used directly, since the items already provide - ways to manage adaptors without having to handle AdaptorPipes. - - :param adaptors: A list of callables to be added as adaptors at - instancing time. - - Methods: - - .. method:: add_adaptor(adaptor, position=None) - - This method is used for adding adaptors to the pipeline given - a certain position. - - :param adaptor: Any callable that works as an adaptor - :param position: An integer meaning the position in which the adaptor - will be inserted. If it's None the adaptor will be appended at - the end of the pipeline. - -Usage -===== - -As it was previously said, in order to use adaptor pipelines you must inherit -your items from the :class:`RobustScrapedItem` class. If you don't know -anything about these items, read the :ref:`topics-items` reference first. - -Once you've created your own item class (inherited from -:class:`RobustScrapedItem`) with the attributes you're going to use, you have -to add adaptor pipelines to each attribute you'd like to adapt data for. For -doing so, RobustScrapedItems provide some useful methods like ``set_adaptors``, -``set_attrib_adaptors``, and more (which are also described in its reference) -so that you don't need to work with :class:`AdaptorPipe` objects directly. - -Adaptors --------- - -Let's now talk a bit about adaptors (singularly), what are them, and how -should they be implemented? - -Adaptors are basically, any callable that receives -a value, modifies it, and returns a new value (or more) so that the next -adaptor goes on with another adapting task (or not). This is done this way to -make the process of modifying information very customizable, and also to make -adaptors reusable, since they are intended to be small functions designed for -simple purposes that can be applied in many different cases. For example, you -could make an adaptor for removing any tags in a text, like this:: - - >>> B_TAG_RE = re.compile(r'') - >>> def remove_b_tags(text): - >>> return B_TAG_RE.sub('', text) - -Then you could easily add this adaptor to a certain attribute's pipeline like -this:: - - >>> item = MyItem() - >>> item.add_adaptor('text', remove_b_tags) - >>> item.attribute('text', u'some random text in bold and some random text in normal font') - >>> item.text - u'some random text in bold and some random text in normal font' - -As you can see, this would make any value that you set to the item through the -``attribute`` method first pass through the ``remove_b_tags`` adaptor, which -would also replace any matching tag with an empty string. - ----- - -But anyway, let's now think of a bit more complicated (and useless) example: -let's say you want to scrape a text, split it into single letters, strip the -vowels, turn the rest to capital letters, and join them again. In this case, -we could use three simple adaptors to process our data, plus a customized -:class:`RobustScrapedItem` for joining single text attributes; let's see an -example:: - - >>> # First of all, we define the item class we're going to use - >>> from string import ascii_letters - >>> from scrapy.contrib.item import RobustScrapedItem - >>> class MyItem(RobustScrapedItem): - >>> ATTRIBUTES = { - >>> 'text': basestring, - >>> } - - >>> def _add_single_attributes(self, attrname, attrtype, attributes): - >>> return ''.join(attributes) - - >>> # Now we'll write the needed adaptors - >>> def to_letters(text): - >>> return tuple(letter for letter in text) - - >>> def is_vowel(letter): - >>> if letter in ascii_letters and letter.lower() not in ('a', 'e', 'i', 'o', 'u'): - >>> return letter - - >>> def to_upper(letter): - >>> return letter.upper() - - >>> # Finally, we'll join all the pieces and see how it works - >>> item = MyItem() - >>> item.set_attrib_adaptors('text', [ - >>> to_letters, - >>> is_vowel, - >>> to_upper, - >>> ]) - -Let's now try with an example text to see what happens:: - - >>> item.attribute('text', 'pi', 'wind', add=True) - >>> item.text - 'PWND' - -More complex adaptors ---------------------- - -Now, after using adaptors a bit, you may find yourself in situations where you need -to use adaptors that receive other parameters from the ``attribute`` method -apart from the value to adapt. - -For example, imagine you have an adaptor that removes certain characters from strings -you provide. Would you make an adaptor for each combination of characters you'd like -to strip? Of course not! - -The way to handle this cases, is to make an adaptor that apart from receiving a value, -as any other adaptor, receives a parameter called ``adaptor_args``. -It's important that the parameter is called this way, since Scrapy finds out whether -an adaptor is able to receive extra parameters or not by making instrospection -and looking for a parameter called this way in the adaptor's parameters list. - -The information this parameter will receive won't be anything else but the same dictionary -of keyword arguments that you pass to the ``attribute`` method when calling it. - -But let's get back to the characters example, how would we implement this? -Quite simmilar to any other adaptor, let's see:: - - def strip_chars(value, adaptor_args): - chars = adaptor_args.get('strip_chars', []) - for char in chars: - value = value.replace(char, '') - return value - -Then, after creating an item and adding the adaptor to one of its pipelines, we could do:: - - >>> item.attribute('text', 'Hi, my name is John', strip_chars=['a', 'i', 'm']) - >>> item.text - 'H, y ne s John' - -Debugging -========= - -While you're coding spiders and adaptors, you usually need to know exactly what -does Scrapy do under the hood with the values you provide. There's a setting -called :setting:``ADAPTORS_DEBUG`` for this purpose that makes Scrapy print -debugging messages each time an adaptors pipeline is run, specifying which -attribute is being adapted data for, the input/output values of each adaptor in -the pipeline, and the input/output of ``_add_single_attributes`` (in some -cases). - -You can enable this setting as any other, either by adding it to your settings -file, or by enabling the environment variable ``SCRAPY_ADAPTORS_DEBUG``. diff --git a/docs/topics/items.rst b/docs/topics/items.rst index c607cf76e..1eee884fa 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -51,144 +51,3 @@ Creating an item and setting its attributes inline:: >>> person ScrapedItem({'age': 23, 'last_name': 'Smith', 'name': 'John'}) -RobustScrapedItems -================== - -.. warning:: - - RobustScapedItems are deprecated and will be replaced by the :ref:`New item - API ` (still in development). - -.. module:: scrapy.contrib.item - :synopsis: Objects for storing scraped data - -.. class:: RobustScrapedItem - - RobustScrapedItems are more complex items (compared to - :class:`ScrapedItem`) and have a few more features available, which - include: - - * Attributes dictionary: items that inherit from RobustScrapedItem are - defined with a dictionary of attributes in the class. This allows the - item to have more logic at the moment of handling and setting attributes - than the :class:`ScrapedItem`. - - * Adaptors: perhaps the most important of the features these items provide. - The adaptors are a system designed for filtering/modifying data before - setting it to the item, that makes cleansing tasks a lot easier. - - * Type checking: RobustScrapedItems come with a built-in type checking - which assures you that no data of the wrong type will get into the items - without raising a warning. - - * Versioning: These items also provide versioning by making a unique hash - for each item based on its attributes values. - - * ItemDeltas: You can subtract two RobustScrapedItems, which allows you to - know the difference between a pair of items. This difference is - represented by a RobustItemDelta object. - -Attributes ----------- - -.. attribute:: RobustScrapedItem.ATTRIBUTES - - This attribute **must** be specified when writing your items, and it's a - dictionary in which the keys are the names of the attributes your item will - have, and their values are the type of those attributes. For multivalued - attributes, you should write the type of the values inside a list, e.g: - ``'numbers': [int]`` - -Methods -------- - -.. method:: RobustScrapedItem.__init__(data=None, adaptor_args=None) - - :param data: Idem as for ScrapedItems - :param adaptor_args: A dictionary of the kind - ``'attribute': [list_of_adaptors]``" for defining adaptors automatically - after instancing the item. - - Constructor of RobustScrapedItem objects. - -.. method:: RobustScrapedItem.attribute(attrname, value, override=False, add=False, ***kwargs) - - Sets the item's ``attrname`` attribute with the given ``value`` filtering - it through the given attribute's adaptor pipeline (if any). - - :param attrname: a string containing the name of the attribute you want - to set. - - :param value: the value you want to assign, which will be adapted by - the corresponding adaptors for the given attribute (if any). - - :param override: if True, makes this method avoid checking if there - was a previous value and sets ``value`` no matter what. - - :param add: if True, tries to concatenate the given ``value`` with the one - already set in the item. For multivalued attributes, this will extend - the list of already-set values, with the new ones. - For single valued attributes, the method _add_single_attributes (which - is explained below) will be called. - - :param kwargs: any extra parameters will be passed in a dictionary to any - adaptor that receives a parameter called ``adaptor_args``. - Check the :ref:`topics-adaptors` topic for more information. - -.. method:: RobustScrapedItem.set_adaptors(adaptors_dict) - - Receives a dict containing a list of adaptors for each desired attribute - (key) and sets each of them as their adaptor pipeline. - -.. method:: RobustScrapedItem.set_attrib_adaptors(attrib, pipe) - - Sets the provided iterable (``pipe``) as the adaptor pipeline for the - given attribute (``attrib``) - -.. method:: RobustScrapedItem.add_adaptor(attrib, adaptor, position=None) - - Adds an adaptor to an already existing (or not) pipeline. - - :param attr: the name of the attribute you're adding adaptors to. - - :param adaptor: a callable to be added to the pipeline. - - :param position: an integer representing the place where to add the adaptor. - If it's ``None``, the adaptor will be appended at the end of the pipeline. - -.. method:: RobustScrapedItem._add_single_attributes(attrname, attrtype, attributes) - - This method is the one to be called whenever a single attribute has to be - joined before storing into an item. That is, - every time you have multiple results at the end of your adaptors pipeline, - and you called the ``attribute`` method with the parameter `add=True`. - - This method is intended to be overriden by you, since by default it - raises an exception. - - :param attrname: the name of the attribute you're setting - :param attrtype: the type of the attribute you're setting - :param attributes: the list of resulting values after the adaptors pipeline - (the one you have to join somehow) - -Examples --------- - -Creating a pretty basic item with a few attributes:: - - from scrapy.contrib.item import RobustScrapedItem - - class MyItem(RobustScrapedItem): - ATTRIBUTES = { - 'name': basestring, - 'size': basestring, - 'colours': [basestring], - } - -Setting some adaptors:: - - -.. note:: - - More RobustScrapedItem examples are about to come. In the meantime, check the :ref:`topics-adaptors` topic to see a few of them. - diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py new file mode 100644 index 000000000..fc709c8ce --- /dev/null +++ b/scrapy/contrib/loader/__init__.py @@ -0,0 +1,93 @@ +""" +Item Loader + +See documentation in docs/topics/loaders.rst +""" + +from collections import defaultdict + +from scrapy.newitem import Item +from scrapy.xpath import HtmlXPathSelector +from scrapy.utils.misc import arg_to_iter +from .common import wrap_loader_context +from .processor import Identity + +class ItemLoader(object): + + default_item_class = Item + default_input_processor = Identity() + default_output_processor = Identity() + + def __init__(self, item=None, **context): + if item is None: + item = self.default_item_class() + self.item = context['item'] = item + self.context = context + self._values = defaultdict(list) + + def add_value(self, field_name, value): + parsed_value = self._parse_input_value(field_name, value) + self._values[field_name] += arg_to_iter(parsed_value) + + def replace_value(self, field_name, value): + parsed_value = self._parse_input_value(field_name, value) + self._values[field_name] = arg_to_iter(parsed_value) + + def populate_item(self): + item = self.item + for field_name in self._values: + item[field_name] = self.get_output_value(field_name) + return item + + def get_output_value(self, field_name): + proc = self.get_output_processor(field_name) + proc = wrap_loader_context(proc, self.context) + return proc(self._values[field_name]) + + def get_collected_values(self, field_name): + return self._values[field_name] + + def get_input_processor(self, field_name): + proc = getattr(self, '%s_in' % field_name, None) + if not proc: + proc = self.item.fields[field_name].get('input_processor', \ + self.default_input_processor) + return proc + + def get_output_processor(self, field_name): + proc = getattr(self, '%s_out' % field_name, None) + if not proc: + proc = self.item.fields[field_name].get('output_processor', \ + self.default_output_processor) + return proc + + def _parse_input_value(self, field_name, value): + proc = self.get_input_processor(field_name) + proc = wrap_loader_context(proc, self.context) + return proc(value) + + +class XPathItemLoader(ItemLoader): + + default_selector_class = HtmlXPathSelector + + def __init__(self, item=None, selector=None, response=None, **context): + if selector is None and response is None: + raise RuntimeError("%s must be instantiated with a selector" \ + "or response" % self.__class__.__name__) + if selector is None: + selector = self.default_selector_class(response) + self.selector = selector + context.update(selector=selector, response=response) + super(XPathItemLoader, self).__init__(item, **context) + + def add_xpath(self, field_name, xpath, re=None): + self.add_value(field_name, self._get_values(field_name, xpath, re)) + + def replace_xpath(self, field_name, xpath, re=None): + self.replace_value(field_name, self._get_values(field_name, xpath, re)) + + def _get_values(self, field_name, xpath, re): + x = self.selector.x(xpath) + return x.re(re) if re else x.extract() + diff --git a/scrapy/contrib/loader/common.py b/scrapy/contrib/loader/common.py new file mode 100644 index 000000000..916524947 --- /dev/null +++ b/scrapy/contrib/loader/common.py @@ -0,0 +1,13 @@ +"""Common functions used in Item Loaders code""" + +from functools import partial +from scrapy.utils.python import get_func_args + +def wrap_loader_context(function, context): + """Wrap functions that receive loader_context to contain the context + "pre-loaded" and expose a interface that receives only one argument + """ + if 'loader_context' in get_func_args(function): + return partial(function, loader_context=context) + else: + return function diff --git a/scrapy/contrib/loader/processor.py b/scrapy/contrib/loader/processor.py new file mode 100644 index 000000000..e883c219d --- /dev/null +++ b/scrapy/contrib/loader/processor.py @@ -0,0 +1,52 @@ +""" +This module provides some commonly used processors for Item Loaders. + +See documentation in docs/topics/loaders.rst +""" + +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.datatypes import MergeDict +from .common import wrap_loader_context + +class ApplyConcat(object): + + def __init__(self, *functions, **default_loader_context): + self.functions = functions + self.default_loader_context = default_loader_context + + def __call__(self, value, loader_context=None): + values = arg_to_iter(value) + if loader_context: + context = MergeDict(loader_context, self.default_loader_context) + else: + context = self.default_loader_context + wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions] + for func in wrapped_funcs: + next_values = [] + for v in values: + next_values += arg_to_iter(func(v)) + values = next_values + return list(values) + + +class TakeFirst(object): + + def __call__(self, values): + for value in values: + if value: + return value + + +class Identity(object): + + def __call__(self, values): + return values + + +class Join(object): + + def __init__(self, separator=u' '): + self.separator = separator + + def __call__(self, values): + return self.separator.join(values) diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py deleted file mode 100644 index f40e4c7e5..000000000 --- a/scrapy/newitem/loader/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -from collections import defaultdict - -from scrapy.utils.datatypes import MergeDict -from scrapy.newitem import Item -from scrapy.newitem.loader.reducers import TakeFirst -from scrapy.newitem.loader.expanders import IdentityExpander -from scrapy.xpath import HtmlXPathSelector - -class Loader(object): - - default_item_class = Item - default_expander = IdentityExpander() - default_reducer = TakeFirst() - - def __init__(self, item=None, **loader_args): - if item is None: - item = self.default_item_class() - self._item = loader_args['item'] = item - self._loader_args = loader_args - self._values = defaultdict(list) - - def add_value(self, field_name, value, **new_loader_args): - self._values[field_name] += self._expand_value(field_name, value, \ - new_loader_args) - - def replace_value(self, field_name, value, **new_loader_args): - self._values[field_name] = self._expand_value(field_name, value, \ - new_loader_args) - - def get_item(self): - item = self._item - for field_name in self._values: - item[field_name] = self.get_reduced_value(field_name) - return item - - def get_expanded_value(self, field_name): - return self._values[field_name] - - def get_reduced_value(self, field_name): - reducer = self.get_reducer(field_name) - return reducer(self._values[field_name]) - - def get_expander(self, field_name): - expander = getattr(self, '%s_exp' % field_name, None) - if not expander: - expander = self._item.fields[field_name].get('expander', \ - self.default_expander) - return expander - - def get_reducer(self, field_name): - reducer = getattr(self, '%s_red' % field_name, None) - if not reducer: - reducer = self._item.fields[field_name].get('reducer', \ - self.default_reducer) - return reducer - - def _expand_value(self, field_name, value, new_loader_args): - loader_args = self._loader_args - if new_loader_args: - loader_args = MergeDict(new_loader_args, self._loader_args) - expander = self.get_expander(field_name) - return expander(value, loader_args=loader_args) - - -class XPathLoader(Loader): - - default_selector_class = HtmlXPathSelector - - def __init__(self, item=None, selector=None, response=None, **loader_args): - if selector is None and response is None: - raise RuntimeError("%s must be instantiated with a selector" \ - "or response" % self.__class__.__name__) - if selector is None: - selector = self.default_selector_class(response) - self.selector = selector - loader_args.update(selector=selector, response=response) - super(XPathLoader, self).__init__(item, **loader_args) - - def add_xpath(self, field_name, xpath, re=None, **new_loader_args): - self.add_value(field_name, self._get_values(field_name, xpath, re), - **new_loader_args) - - def replace_xpath(self, field_name, xpath, re=None, **new_loader_args): - self.replace_value(field_name, self._get_values(field_name, xpath, re), \ - **new_loader_args) - - def _get_values(self, field_name, xpath, re): - x = self.selector.x(xpath) - return x.re(re) if re else x.extract() diff --git a/scrapy/newitem/loader/expanders.py b/scrapy/newitem/loader/expanders.py deleted file mode 100644 index 3e7d77e73..000000000 --- a/scrapy/newitem/loader/expanders.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -This module provides some commonly used Expanders. - -See documentation in docs/topics/newitem-loader.rst -""" - -from scrapy.utils.misc import arg_to_iter -from scrapy.utils.python import get_func_args -from scrapy.utils.datatypes import MergeDict - -class TreeExpander(object): - - def __init__(self, *functions, **default_loader_args): - self.default_loader_args = default_loader_args - self.wrapped_funcs = [] - for func in functions: - if 'loader_args' in get_func_args(func): - wfunc = self.wrap_with_args(func) - else: - wfunc = self.wrap_no_args(func) - self.wrapped_funcs.append(wfunc) - - def wrap_no_args(self, f): - return lambda x, y: f(x) - - def wrap_with_args(self, f): - return lambda x, y: f(x, loader_args=y) - - def __call__(self, value, loader_args=None): - values = arg_to_iter(value) - largs = self.default_loader_args - if loader_args: - largs = MergeDict(loader_args, self.default_loader_args) - for func in self.wrapped_funcs: - next_values = [] - for v in values: - next_values += arg_to_iter(func(v, largs)) - values = next_values - return list(values) - - -class IdentityExpander(object): - - def __call__(self, values, loader_args): - return arg_to_iter(values) diff --git a/scrapy/newitem/loader/reducers.py b/scrapy/newitem/loader/reducers.py deleted file mode 100644 index 8f9a3cc9b..000000000 --- a/scrapy/newitem/loader/reducers.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -This module provides some commonly used Reducers. - -See documentation in docs/topics/newitem-loader.rst -""" - -class TakeFirst(object): - - def __call__(self, values): - for value in values: - if value: - return value - - -class Identity(object): - - def __call__(self, values): - return values - - -class Join(object): - - def __init__(self, separator=u' '): - self.separator = separator - - def __call__(self, values): - return self.separator.join(values) diff --git a/scrapy/tests/test_contrib_loader.py b/scrapy/tests/test_contrib_loader.py new file mode 100644 index 000000000..d7e36251e --- /dev/null +++ b/scrapy/tests/test_contrib_loader.py @@ -0,0 +1,280 @@ +import unittest + +from scrapy.contrib.loader import ItemLoader, XPathItemLoader +from scrapy.contrib.loader.processor import ApplyConcat, Join, Identity +from scrapy.newitem import Item, Field +from scrapy.xpath import HtmlXPathSelector +from scrapy.http import HtmlResponse + +# test items + +class NameItem(Item): + name = Field() + +class TestItem(NameItem): + url = Field() + summary = Field() + +# test item loaders + +class NameItemLoader(ItemLoader): + default_item_class = TestItem + +class TestItemLoader(NameItemLoader): + name_in = ApplyConcat(lambda v: v.title()) + +class DefaultedItemLoader(NameItemLoader): + default_input_processor = ApplyConcat(lambda v: v[:-1]) + +# test processors + +def processor_with_args(value, other=None, loader_context=None): + if 'key' in loader_context: + return loader_context['key'] + return value + +class ItemLoaderTest(unittest.TestCase): + + def test_populate_item_using_default_loader(self): + i = TestItem() + i['summary'] = u'lala' + ip = ItemLoader(item=i) + ip.add_value('name', u'marta') + item = ip.populate_item() + assert item is i + self.assertEqual(item['summary'], u'lala') + self.assertEqual(item['name'], [u'marta']) + + def test_populate_item_using_custom_loader(self): + ip = TestItemLoader() + ip.add_value('name', u'marta') + item = ip.populate_item() + self.assertEqual(item['name'], [u'Marta']) + + def test_add_value(self): + ip = TestItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_collected_values('name'), [u'Marta']) + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + ip.add_value('name', u'pepe') + self.assertEqual(ip.get_collected_values('name'), [u'Marta', u'Pepe']) + self.assertEqual(ip.get_output_value('name'), [u'Marta', u'Pepe']) + + def test_replace_value(self): + ip = TestItemLoader() + ip.replace_value('name', u'marta') + self.assertEqual(ip.get_collected_values('name'), [u'Marta']) + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + ip.replace_value('name', u'pepe') + self.assertEqual(ip.get_collected_values('name'), [u'Pepe']) + self.assertEqual(ip.get_output_value('name'), [u'Pepe']) + + def test_apply_concat_filter(self): + def filter_world(x): + return None if x == 'world' else x + + proc = ApplyConcat(filter_world, str.upper) + self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']), + ['HELLO', 'THIS', 'IS', 'SCRAPY']) + + def test_map_concat_filter_multiple_functions(self): + class TestItemLoader(NameItemLoader): + name_in = ApplyConcat(lambda v: v.title(), lambda v: v[:-1]) + + ip = TestItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Mart']) + item = ip.populate_item() + self.assertEqual(item['name'], [u'Mart']) + + def test_default_input_processor(self): + ip = DefaultedItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mart']) + + def test_inherited_default_input_processor(self): + class InheritDefaultedItemLoader(DefaultedItemLoader): + pass + + ip = InheritDefaultedItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mart']) + + def test_input_processor_inheritance(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(lambda v: v.lower()) + + ip = ChildItemLoader() + ip.add_value('url', u'HTTP://scrapy.ORG') + self.assertEqual(ip.get_output_value('url'), [u'http://scrapy.org']) + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + + class ChildChildItemLoader(ChildItemLoader): + url_in = ApplyConcat(lambda v: v.upper()) + summary_in = ApplyConcat(lambda v: v) + + ip = ChildChildItemLoader() + ip.add_value('url', u'http://scrapy.org') + self.assertEqual(ip.get_output_value('url'), [u'HTTP://SCRAPY.ORG']) + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'Marta']) + + def test_empty_map_concat(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): + name_in = ApplyConcat() + + ip = IdentityDefaultedItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'marta']) + + def test_identity_input_processor(self): + class IdentityDefaultedItemLoader(DefaultedItemLoader): + name_in = Identity() + + ip = IdentityDefaultedItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'marta']) + + def test_extend_custom_input_processors(self): + class ChildItemLoader(TestItemLoader): + name_in = ApplyConcat(TestItemLoader.name_in, unicode.swapcase) + + ip = ChildItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'mARTA']) + + def test_extend_default_input_processors(self): + class ChildDefaultedItemLoader(DefaultedItemLoader): + name_in = ApplyConcat(DefaultedItemLoader.default_input_processor, unicode.swapcase) + + ip = ChildDefaultedItemLoader() + ip.add_value('name', u'marta') + self.assertEqual(ip.get_output_value('name'), [u'MART']) + + def test_output_processor_using_function(self): + ip = TestItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemLoader(TestItemLoader): + name_out = u" ".join + + ip = TakeFirstItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar Ta') + + def test_output_processor_using_classes(self): + ip = TestItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class TakeFirstItemLoader(TestItemLoader): + name_out = Join() + + ip = TakeFirstItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar Ta') + + class TakeFirstItemLoader(TestItemLoader): + name_out = Join("
") + + ip = TakeFirstItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), u'Mar
Ta') + + def test_default_output_processor(self): + ip = TestItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + class LalaItemLoader(TestItemLoader): + default_output_processor = Identity() + + ip = LalaItemLoader() + ip.add_value('name', [u'mar', u'ta']) + self.assertEqual(ip.get_output_value('name'), [u'Mar', u'Ta']) + + def test_loader_context_on_declaration(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor_with_args, key=u'val') + + ip = ChildItemLoader() + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_loader_context_on_instantiation(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor_with_args) + + ip = ChildItemLoader(key=u'val') + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_loader_context_on_assign(self): + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor_with_args) + + ip = ChildItemLoader() + ip.context['key'] = u'val' + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['val']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['val']) + + def test_item_passed_to_input_processor_functions(self): + def processor(value, loader_context): + return loader_context['item']['name'] + + class ChildItemLoader(TestItemLoader): + url_in = ApplyConcat(processor) + + it = TestItem(name='marta') + ip = ChildItemLoader(item=it) + ip.add_value('url', u'text') + self.assertEqual(ip.get_output_value('url'), ['marta']) + ip.replace_value('url', u'text2') + self.assertEqual(ip.get_output_value('url'), ['marta']) + + def test_add_value_on_unknown_field(self): + ip = TestItemLoader() + self.assertRaises(KeyError, ip.add_value, 'wrong_field', [u'lala', u'lolo']) + + +class TestXPathItemLoader(XPathItemLoader): + default_item_class = TestItem + name_in = ApplyConcat(lambda v: v.title()) + +class XPathItemLoaderTest(unittest.TestCase): + + def test_constructor_errors(self): + self.assertRaises(RuntimeError, XPathItemLoader) + + def test_constructor_with_selector(self): + sel = HtmlXPathSelector(text=u"
marta
") + l = TestXPathItemLoader(selector=sel) + self.assert_(l.selector is sel) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'Marta']) + + def test_constructor_with_response(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathItemLoader(response=response) + self.assert_(l.selector) + l.add_xpath('name', '//div/text()') + self.assertEqual(l.get_output_value('name'), [u'Marta']) + + def test_add_xpath_re(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathItemLoader(response=response) + l.add_xpath('name', '//div/text()', re='ma') + self.assertEqual(l.get_output_value('name'), [u'Ma']) + + +if __name__ == "__main__": + unittest.main() + diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py deleted file mode 100644 index a32d2b036..000000000 --- a/scrapy/tests/test_itemloader.py +++ /dev/null @@ -1,272 +0,0 @@ -import unittest - -from scrapy.newitem.loader import Loader, XPathLoader -from scrapy.newitem.loader.expanders import TreeExpander, IdentityExpander -from scrapy.newitem.loader.reducers import Join, Identity -from scrapy.newitem import Item, Field -from scrapy.xpath import HtmlXPathSelector -from scrapy.http import HtmlResponse - -# test items - -class NameItem(Item): - name = Field() - -class TestItem(NameItem): - url = Field() - summary = Field() - -# test loaders - -class NameLoader(Loader): - default_item_class = TestItem - -class TestLoader(NameLoader): - name_exp = TreeExpander(lambda v: v.title()) - -class DefaultedLoader(NameLoader): - default_expander = TreeExpander(lambda v: v[:-1]) - -# test expanders - -def expander_func_with_args(value, other=None, loader_args=None): - if 'key' in loader_args: - return loader_args['key'] - return value - -class LoaderTest(unittest.TestCase): - - def test_get_item_using_default_loader(self): - i = TestItem() - i['summary'] = u'lala' - il = Loader(item=i) - il.add_value('name', u'marta') - item = il.get_item() - assert item is i - self.assertEqual(item['summary'], u'lala') - self.assertEqual(item['name'], u'marta') - - def test_get_item_using_custom_loader(self): - il = TestLoader() - il.add_value('name', u'marta') - item = il.get_item() - self.assertEqual(item['name'], u'Marta') - - def test_add_value(self): - il = TestLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_expanded_value('name'), [u'Marta']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - il.add_value('name', u'pepe') - self.assertEqual(il.get_expanded_value('name'), [u'Marta', u'Pepe']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - def test_replace_value(self): - il = TestLoader() - il.replace_value('name', u'marta') - self.assertEqual(il.get_expanded_value('name'), [u'Marta']) - self.assertEqual(il.get_reduced_value('name'), u'Marta') - il.replace_value('name', u'pepe') - self.assertEqual(il.get_expanded_value('name'), [u'Pepe']) - self.assertEqual(il.get_reduced_value('name'), u'Pepe') - - def test_tree_expander_filter(self): - def filter_world(x): - return None if x == 'world' else x - - expander = TreeExpander(filter_world, str.upper) - self.assertEqual(expander(['hello', 'world', 'this', 'is', 'scrapy']), - ['HELLO', 'THIS', 'IS', 'SCRAPY']) - - def test_tree_expander_multiple_functions(self): - class TestLoader(NameLoader): - name_exp = TreeExpander(lambda v: v.title(), lambda v: v[:-1]) - - il = TestLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Mart') - item = il.get_item() - self.assertEqual(item['name'], u'Mart') - - def test_default_expander(self): - il = DefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mart') - - def test_inherited_default_expander(self): - class InheritDefaultedLoader(DefaultedLoader): - pass - - il = InheritDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mart') - - def test_expander_inheritance(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(lambda v: v.lower()) - - il = ChildLoader() - il.add_value('url', u'HTTP://scrapy.ORG') - self.assertEqual(il.get_reduced_value('url'), u'http://scrapy.org') - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - class ChildChildLoader(ChildLoader): - url_exp = TreeExpander(lambda v: v.upper()) - summary_exp = TreeExpander(lambda v: v) - - il = ChildChildLoader() - il.add_value('url', u'http://scrapy.org') - self.assertEqual(il.get_reduced_value('url'), u'HTTP://SCRAPY.ORG') - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'Marta') - - def test_empty_tree_expander(self): - class IdentityDefaultedLoader(DefaultedLoader): - name_exp = TreeExpander() - - il = IdentityDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'marta') - - def test_identity_expander(self): - class IdentityDefaultedLoader(DefaultedLoader): - name_exp = IdentityExpander() - - il = IdentityDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'marta') - - def test_extend_custom_expanders(self): - class ChildLoader(TestLoader): - name_exp = TreeExpander(TestLoader.name_exp, unicode.swapcase) - - il = ChildLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'mARTA') - - def test_extend_default_expanders(self): - class ChildDefaultedLoader(DefaultedLoader): - name_exp = TreeExpander(DefaultedLoader.default_expander, unicode.swapcase) - - il = ChildDefaultedLoader() - il.add_value('name', u'marta') - self.assertEqual(il.get_reduced_value('name'), u'MART') - - def test_reducer_using_function(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class TakeFirstLoader(TestLoader): - name_red = u" ".join - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - - def test_reducer_using_classes(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class TakeFirstLoader(TestLoader): - name_red = Join() - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar Ta') - - class TakeFirstLoader(TestLoader): - name_red = Join("
") - - il = TakeFirstLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar
Ta') - - def test_default_reducer(self): - il = TestLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), u'Mar') - - class LalaLoader(TestLoader): - default_reducer = Identity() - - il = LalaLoader() - il.add_value('name', [u'mar', u'ta']) - self.assertEqual(il.get_reduced_value('name'), [u'Mar', u'Ta']) - - def test_expander_args_on_declaration(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args, key=u'val') - - il = ChildLoader() - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_expander_args_on_instantiation(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args) - - il = ChildLoader(key=u'val') - il.add_value('url', u'text') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_expander_args_on_assign(self): - class ChildLoader(TestLoader): - url_exp = TreeExpander(expander_func_with_args) - - il = ChildLoader() - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'val') - - def test_item_passed_to_expander_functions(self): - def exp_func(value, loader_args): - return loader_args['item']['name'] - - class ChildLoader(TestLoader): - url_exp = TreeExpander(exp_func) - - it = TestItem(name='marta') - il = ChildLoader(item=it) - il.add_value('url', u'text', key=u'val') - self.assertEqual(il.get_reduced_value('url'), 'marta') - - def test_add_value_on_unknown_field(self): - il = TestLoader() - self.assertRaises(KeyError, il.add_value, 'wrong_field', [u'lala', u'lolo']) - - -class TestXPathLoader(XPathLoader): - default_item_class = TestItem - name_exp = TreeExpander(lambda v: v.title()) - -class XPathLoaderTest(unittest.TestCase): - - def test_constructor_errors(self): - self.assertRaises(RuntimeError, XPathLoader) - - def test_constructor_with_selector(self): - sel = HtmlXPathSelector(text=u"
marta
") - l = TestXPathLoader(selector=sel) - self.assert_(l.selector is sel) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_reduced_value('name'), u'Marta') - - def test_constructor_with_response(self): - response = HtmlResponse(url="", body="
marta
") - l = TestXPathLoader(response=response) - self.assert_(l.selector) - l.add_xpath('name', '//div/text()') - self.assertEqual(l.get_reduced_value('name'), u'Marta') - - def test_add_xpath_re(self): - response = HtmlResponse(url="", body="
marta
") - l = TestXPathLoader(response=response) - l.add_xpath('name', '//div/text()', re='ma') - self.assertEqual(l.get_reduced_value('name'), u'Ma') - - -if __name__ == "__main__": - unittest.main() - diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 62ca3be86..1f7c39773 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -54,6 +54,14 @@ class ResponseUtilsTest(unittest.TestCase): response = Response(url='http://example.org', body=body) self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + # meta refresh in multiple lines + body = """ + """ + response = Response(url='http://example.org', body=body) + self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage')) + def test_response_httprepr(self): r1 = Response("http://www.example.com") self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 5169b990b..17d204377 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -31,7 +31,7 @@ def get_base_url(response): response.cache['base_url'] = match.group(1) if match else response.url return response.cache['base_url'] -META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.IGNORECASE) +META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE) def get_meta_refresh(response): """ Return a tuple of two strings containing the interval and url included in the http-equiv parameter of the HTML meta element. If no url is included