From c508f406892f9d38860fedf1caf8a41fc69bc184 Mon Sep 17 00:00:00 2001 From: Elias Dorneles Date: Thu, 15 Sep 2016 18:05:09 -0300 Subject: [PATCH] use harcoded URLs, remove item reference on second spider --- docs/intro/tutorial.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 0a3361799..d160bfc5c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -100,10 +100,12 @@ This is the code for our first Spider; save it in a file named name = "quotes" def start_requests(self): - base_url = 'http://quotes.toscrape.com' - for path in ['/page/1/', '/page/2/']: - yield scrapy.Request(url=base_url + path, - callback=self.parse) + urls = [ + 'http://quotes.toscrape.com/page/1/', + 'http://quotes.toscrape.com/page/2/', + ] + for url in urls: + yield scrapy.Request(url=url, callback=self.parse) def parse(self, response): page = response.url.split("/")[-2] @@ -397,7 +399,6 @@ want for all of them? Here is a modification to our spider that does just that:: import scrapy - from tutorial.items import QuoteItem class QuotesSpider(scrapy.Spider): @@ -408,12 +409,13 @@ Here is a modification to our spider that does just that:: def parse(self, response): for quote in response.xpath('//div[@class="quote"]'): - item = QuoteItem() - item['text'] = quote.xpath('span[@class="text"]/text()').extract_first() - item['author'] = quote.xpath('span/small/text()').extract_first() - yield item + yield { + 'text': quote.xpath('span[@class="text"]/text()').extract_first(), + 'author': quote.xpath('span/small/text()').extract_first(), + } + next_page = response.xpath('//li[@class="next"]/a/@href').extract_first() - if next_page: + if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse)