From ca27010d4f8c35d1c98259e56be797a18b044304 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 15 Aug 2018 16:22:56 +0500 Subject: [PATCH] DOC .extract_first() -> .get() --- docs/intro/overview.rst | 6 ++--- docs/intro/tutorial.rst | 49 +++++++++++++++++++++-------------------- docs/topics/shell.rst | 2 +- docs/topics/spiders.rst | 10 ++++----- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6f1c2c43f..9d7c94d39 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.xpath('span/small/text()').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.xpath('span/small/text()').get(), } - next_page = response.css('li.next a::attr("href")').extract_first() + next_page = response.css('li.next a::attr("href")').get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fa6dc274d..a24cf0f5b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -269,23 +269,24 @@ The other thing is that the result of calling ``.extract()`` is a list, because we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When you know you just want the first result, as in this case, you can do:: - >>> response.css('title::text').extract_first() + >>> response.css('title::text').get() 'Quotes to Scrape' As an alternative, you could've written:: - >>> response.css('title::text')[0].extract() + >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.extract_first()`` avoids an ``IndexError`` and returns -``None`` when it doesn't find any element matching the selection. +However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` +instance avoids an ``IndexError`` and returns ``None`` when it doesn't +find any element matching the selection. There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail to be scraped, you can at least get **some** data. Besides the :meth:`~scrapy.selector.Selector.extract` and -:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use +:meth:`~scrapy.selector.SelectorList.get` methods, you can also use the :meth:`~scrapy.selector.Selector.re` method to extract using `regular expressions`:: @@ -314,7 +315,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: >>> response.xpath('//title') [] - >>> response.xpath('//title/text()').extract_first() + >>> response.xpath('//title/text()').get() 'Quotes to Scrape' XPath expressions are very powerful, and are the foundation of Scrapy @@ -383,10 +384,10 @@ variable, so that we can run our CSS selectors directly on a particular quote:: Now, let's extract ``title``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").extract_first() + >>> title = quote.css("span.text::text").get() >>> title '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' - >>> author = quote.css("small.author::text").extract_first() + >>> author = quote.css("small.author::text").get() >>> author 'Albert Einstein' @@ -401,8 +402,8 @@ Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): - ... text = quote.css("span.text::text").extract_first() - ... author = quote.css("small.author::text").extract_first() + ... text = quote.css("span.text::text").get() + ... author = quote.css("small.author::text").get() ... tags = quote.css("div.tags a.tag::text").extract() ... print(dict(text=text, author=author, tags=tags)) {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} @@ -434,8 +435,8 @@ in the callback, as you can see below:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), 'tags': quote.css('div.tags a.tag::text').extract(), } @@ -508,14 +509,14 @@ markup: We can try extracting it in the shell:: - >>> response.css('li.next a').extract_first() + >>> response.css('li.next a').get() 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, Scrapy supports a CSS extension that let's you select the attribute contents, like this:: - >>> response.css('li.next a::attr(href)').extract_first() + >>> response.css('li.next a::attr(href)').get() '/page/2/' Let's see now our spider modified to recursively follow the link to the next @@ -533,12 +534,12 @@ page, extracting data from it:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), 'tags': quote.css('div.tags a.tag::text').extract(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -584,12 +585,12 @@ As a shortcut for creating Request objects you can use def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('span small::text').get(), 'tags': quote.css('div.tags a.tag::text').extract(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -641,7 +642,7 @@ this time for scraping author information:: def parse_author(self, response): def extract_with_css(query): - return response.css(query).extract_first().strip() + return response.css(query).get().strip() yield { 'name': extract_with_css('h3.author-title::text'), @@ -710,11 +711,11 @@ with a specific tag, building the URL based on the argument:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 11ab199f2..9de6abef7 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -179,7 +179,7 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: - >>> response.xpath('//title/text()').extract_first() + >>> response.xpath('//title/text()').get() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("https://reddit.com") diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 697732b47..4505b7315 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules:: self.logger.info('Hi, this is an item page! %s', response.url) item = scrapy.Item() item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = response.xpath('//td[@id="item_name"]/text()').extract() - item['description'] = response.xpath('//td[@id="item_description"]/text()').extract() + item['name'] = response.xpath('//td[@id="item_name"]/text()').get() + item['description'] = response.xpath('//td[@id="item_description"]/text()').get() return item @@ -548,9 +548,9 @@ These spiders are pretty easy to use, let's have a look at one example:: self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract())) item = TestItem() - item['id'] = node.xpath('@id').extract() - item['name'] = node.xpath('name').extract() - item['description'] = node.xpath('description').extract() + item['id'] = node.xpath('@id').get() + item['name'] = node.xpath('name').get() + item['description'] = node.xpath('description').get() return item Basically what we did up there was to create a spider that downloads a feed from