diff --git a/docs/experimental/loaders.rst b/docs/experimental/loaders.rst index dc48ecbdd..dfec482d2 100644 --- a/docs/experimental/loaders.rst +++ b/docs/experimental/loaders.rst @@ -145,7 +145,7 @@ Declaring Extenders and Reducers As seen in the previous section, extenders and reducers can be declared in the Loader definition, and it's very common to declare expanders this way. However, there is one more place where you can specify the exanders and reducers to use: -in the :ref:`Item Field ` metadata. Here is an example:: from scrapy.newitem import Item, Field from scrapy.newitem.loader.expanders import TreeExpander @@ -301,25 +301,47 @@ Loader objects in which case this argument is ignored. :type response: :class:`~scrapy.http.Response` object - .. method:: add_xpath(field_name, xpath, \**new_loader_args) + .. method:: add_xpath(field_name, xpath, re=None, \**new_loader_args) Similar to :meth:`Loader.add_value` but receives an XPath instead of a value, which is used to extract a list of unicode strings from the - selector associated with this :class:`XPathLoader`. + selector associated with this :class:`XPathLoader`. If the ``re`` + argument is given, it's used for extrating data from the selector using + the :meth:`~scrapy.xpath.XPathSelector.re` method. - Example:: + :param xpath: the XPath to extract data from + :type xpath: str + :param re: a regular expression to use for extracting data from the + selected XPath region + :type re: str or compiled regex + + Examples:: + + # HTML snippet:

Color TV

loader.add_xpath('name', '//p[@class="product-name"]') + # HTML snippet:

the price is $1200

+ loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)') - .. method:: replace_xpath(field_name, xpath, \**new_loader_args) + .. method:: replace_xpath(field_name, xpath, re=None, \**new_loader_args) Similar to :meth:`add_xpath` but replaces collected data instead of adding it. .. attribute:: default_selector_class - The class used to construct the selector, if only a response is given - in the constructor + The class used to construct the :attr:`selector` of this + :class:`XPathLoader`, if only a response is given in the constructor. + If a selector is given in the constructor this attribute is ignored. + This attribute is sometimes overridden in subclasses. + + .. attribute:: selector + + The :class:`~scrapy.xpath.XPathSelector` object to extract data from. + It's either the selector given in the constructor or one created from + the response given in the constructor using the + :attr:`default_selector_class`. This attribute is meant to be + read-only. .. _topics-loader-extending: diff --git a/scrapy/newitem/loader/__init__.py b/scrapy/newitem/loader/__init__.py index 9edebf061..f40e4c7e5 100644 --- a/scrapy/newitem/loader/__init__.py +++ b/scrapy/newitem/loader/__init__.py @@ -72,14 +72,18 @@ class XPathLoader(Loader): "or response" % self.__class__.__name__) if selector is None: selector = self.default_selector_class(response) - self._selector = selector + self.selector = selector loader_args.update(selector=selector, response=response) super(XPathLoader, self).__init__(item, **loader_args) - def add_xpath(self, field_name, xpath, **new_loader_args): - self.add_value(field_name, self._selector.x(xpath).extract(), \ + def add_xpath(self, field_name, xpath, re=None, **new_loader_args): + self.add_value(field_name, self._get_values(field_name, xpath, re), **new_loader_args) - def replace_xpath(self, field_name, xpath, **new_loader_args): - self.replace_value(field_name, self._selector.x(xpath).extract(), \ + def replace_xpath(self, field_name, xpath, re=None, **new_loader_args): + self.replace_value(field_name, self._get_values(field_name, xpath, re), \ **new_loader_args) + + def _get_values(self, field_name, xpath, re): + x = self.selector.x(xpath) + return x.re(re) if re else x.extract() diff --git a/scrapy/tests/test_itemloader.py b/scrapy/tests/test_itemloader.py index eefcf6421..a32d2b036 100644 --- a/scrapy/tests/test_itemloader.py +++ b/scrapy/tests/test_itemloader.py @@ -249,14 +249,24 @@ class XPathLoaderTest(unittest.TestCase): def test_constructor_with_selector(self): sel = HtmlXPathSelector(text=u"
marta
") l = TestXPathLoader(selector=sel) + self.assert_(l.selector is sel) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_reduced_value('name'), u'Marta') def test_constructor_with_response(self): response = HtmlResponse(url="", body="
marta
") l = TestXPathLoader(response=response) + self.assert_(l.selector) l.add_xpath('name', '//div/text()') self.assertEqual(l.get_reduced_value('name'), u'Marta') + def test_add_xpath_re(self): + response = HtmlResponse(url="", body="
marta
") + l = TestXPathLoader(response=response) + l.add_xpath('name', '//div/text()', re='ma') + self.assertEqual(l.get_reduced_value('name'), u'Ma') + + if __name__ == "__main__": unittest.main() +