diff --git a/scrapy/trunk/docs/tutorial.rst b/scrapy/trunk/docs/tutorial.rst index 09524ba46..43a6bff15 100644 --- a/scrapy/trunk/docs/tutorial.rst +++ b/scrapy/trunk/docs/tutorial.rst @@ -264,10 +264,10 @@ that property here, so:: sites = hxs.x('//ul[2]/li') for site in sites: - title = site.x('a/text()').extract() - link = site.x('a/@href').extract() - desc = site.x('text()').extract() - print title, link, desc + title = site.x('a/text()').extract() + link = site.x('a/@href').extract() + desc = site.x('text()').extract() + print title, link, desc Let's add this code to our spider:: @@ -286,10 +286,10 @@ Let's add this code to our spider:: hxs = HtmlXPathSelector(response) sites = hxs.x('//ul[2]/li') for site in sites: - title = site.x('a/text()').extract() - link = site.x('a/@href').extract() - desc = site.x('text()').extract() - print title, link, desc + title = site.x('a/text()').extract() + link = site.x('a/@href').extract() + desc = site.x('text()').extract() + print title, link, desc return [] SPIDER = OpenDirectorySpider() @@ -306,5 +306,41 @@ In Scrapy, items are the placeholder to use for the scraped data. They are represented by a ScrapedItem object, or any descendant class instance, and store the information in class attributes +Spiders are supposed to return their scraped data in the form of ScrapedItems, +so to actually return the data we've scraped so far, the code for our spider +should be like this:: + + from scrapy.spider import BaseSpider + from scrapy.item import ScrapedItem + from scrapy.xpath.selector import HtmlXPathSelector + + + class OpenDirectorySpider(BaseSpider): + domain_name = "dmoz.org" + start_urls = [ + "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", + "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/" + ] + + def parse(self, response): + hxs = HtmlXPathSelector(response) + sites = hxs.x('//ul[2]/li') + items = [] + for site in sites: + item = ScrapedItem() + item.title = site.x('a/text()').extract() + item.link = site.x('a/@href').extract() + item.desc = site.x('text()').extract() + items.append(item) + return items + + SPIDER = OpenDirectorySpider() + +Now doing a crawl on the dmoz.org domain yields ScrapedItems:: + + [dmoz/dmoz.org] DEBUG: Scraped ScrapedItem({'title': [u'Text Processing in Python'], 'link': [u'http://gnosis.cx/TPiP/'], 'desc': [u' - By David Mertz; Addison Wesley. Book in progress, full text, ASCII format. Asks for feedback. [author website, Gnosis Software, Inc.]\n']}) in + [dmoz/dmoz.org] DEBUG: Scraped ScrapedItem({'title': [u'XML Processing with Python'], 'link': [u'http://www.informit.com/store/product.aspx?isbn=0130211192'], 'desc': [u' - By Sean McGrath; Prentice Hall PTR, 2000, ISBN 0130211192, has CD-ROM. Methods to build XML applications fast, Python tutorial, DOM and SAX, new Pyxie open source XML processing library. [Prentice Hall PTR]\n']}) in + + Item Pipeline =============