diff --git a/docs/experimental/index.rst b/docs/experimental/index.rst index 1d1bd4f8e..5a8a61a58 100644 --- a/docs/experimental/index.rst +++ b/docs/experimental/index.rst @@ -19,6 +19,5 @@ it's properly merged) . Use at your own risk. .. toctree:: :maxdepth: 1 - loaders exporters images diff --git a/docs/topics/index.rst b/docs/topics/index.rst index 500d7f612..3717a79b9 100644 --- a/docs/topics/index.rst +++ b/docs/topics/index.rst @@ -14,8 +14,9 @@ Chapter 1. Main topics architecture shell spiders - selectors items + loaders + selectors request-response Chapter 2. Extension Mechanisms diff --git a/docs/experimental/loaders.rst b/docs/topics/loaders.rst similarity index 100% rename from docs/experimental/loaders.rst rename to docs/topics/loaders.rst diff --git a/examples/googledir/googledir/items.py b/examples/googledir/googledir/items.py index 4685dc82a..c5d11b2cd 100644 --- a/examples/googledir/googledir/items.py +++ b/examples/googledir/googledir/items.py @@ -8,4 +8,3 @@ class GoogledirItem(Item): def __str__(self): return "Google Category: name=%s url=%s" % (self['name'], self['url']) - diff --git a/examples/googledir/googledir/spiders/google_directory.py b/examples/googledir/googledir/spiders/google_directory.py index c46e6c6ce..30221ad38 100644 --- a/examples/googledir/googledir/spiders/google_directory.py +++ b/examples/googledir/googledir/spiders/google_directory.py @@ -1,6 +1,7 @@ from scrapy.xpath import HtmlXPathSelector from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule +from scrapy.contrib.loader import XPathItemLoader from googledir.items import GoogledirItem class GoogleDirectorySpider(CrawlSpider): @@ -16,18 +17,25 @@ class GoogleDirectorySpider(CrawlSpider): ) def parse_category(self, response): - # The selector we're going to use in order to extract data from the page - hxs = HtmlXPathSelector(response) + # The main selector we're using to extract data from the page + main_selector = HtmlXPathSelector(response) - # The path to website links in directory page - links = hxs.select('//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font') + # The XPath to website links in the directory page + xpath = '//td[descendant::a[contains(@href, "#pagerank")]]/following-sibling::td/font' - for link in links: + # Get a list of (sub) selectors to each website node pointed by the XPath + sub_selectors = main_selector.select(xpath) + + # Iterate over the sub-selectors to extract data for each website + for selector in sub_selectors: item = GoogledirItem() - item['name'] = link.select('a/text()').extract() - item['url'] = link.select('a/@href').extract() - item['description'] = link.select('font[2]/text()').extract() - yield item + l = XPathItemLoader(item=item, selector=selector) + l.add_xpath('name', 'a/text()') + l.add_xpath('url', 'a/@href') + l.add_xpath('description', 'font[2]/text()') + + # Here we populate the item and yield it + yield l.load_item() SPIDER = GoogleDirectorySpider() diff --git a/scrapy/contrib/loader/__init__.py b/scrapy/contrib/loader/__init__.py index 7e5275386..b91516696 100644 --- a/scrapy/contrib/loader/__init__.py +++ b/scrapy/contrib/loader/__init__.py @@ -75,7 +75,7 @@ class XPathItemLoader(ItemLoader): def __init__(self, item=None, selector=None, response=None, **context): if selector is None and response is None: - raise RuntimeError("%s must be instantiated with a selector" \ + raise RuntimeError("%s must be instantiated with a selector " \ "or response" % self.__class__.__name__) if selector is None: selector = self.default_selector_class(response)