From 91f4d6dc5105b187bc7a6a3b00581e7d5c839fe8 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 18 Feb 2010 16:51:05 -0200 Subject: [PATCH] docs: adds another spider example that yields multiples requests/items from a single callback --- docs/topics/spiders.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index a658f7e4f..e60549098 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -169,6 +169,31 @@ Let's see an example:: SPIDER = MySpider() +Another example returning multiples Requests and Items from a single callback:: + + from scrapy.selector import HtmlXPathSelector + from scrapy.spider import BaseSpider + from scrapy.http import Request + from myproject.items import MyItem + + class MySpider(BaseSpider): + domain_name = 'http://www.example.com' + start_urls = [ + 'http://www.example.com/1.html', + 'http://www.example.com/2.html', + 'http://www.example.com/3.html', + ] + + def parse(self, response): + hxs = HtmlXPathSelector(response) + for h3 in hxs.select('//h3').extract(): + yield MyItem(title=h3) + + for url in hxs.select('//a/@href').extract(): + yield Request(url, callback=self.parse) + + SPIDER = MySpider() + .. module:: scrapy.contrib.spiders :synopsis: Collection of generic spiders