diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 07fb4807f..46e84b21c 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -254,7 +254,7 @@ data. To extract the text from the title above, you can do:: - >>> response.css('title::text').extract() + >>> response.css('title::text').getall() ['Quotes to Scrape'] There are two things to note here: one is that we've added ``::text`` to the @@ -262,12 +262,12 @@ CSS query, to mean we want to select only the text elements directly inside ```` element. If we don't specify ``::text``, we'd get the full title element, including its tags:: - >>> response.css('title').extract() + >>> response.css('title').getall() ['<title>Quotes to Scrape'] -The other thing is that the result of calling ``.extract()`` is a list, because -we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When -you know you just want the first result, as in this case, you can do:: +The other thing is that the result of calling ``.getall()`` is a list: it is +possible that a selector returns more than one result, so we extract them all. +When you know you just want the first result, as in this case, you can do:: >>> response.css('title::text').get() 'Quotes to Scrape' @@ -392,10 +392,10 @@ using the ``quote`` object we just created:: >>> author 'Albert Einstein' -Given that the tags are a list of strings, we can use the ``.extract()`` method +Given that the tags are a list of strings, we can use the ``.getall()`` method to get all of them:: - >>> tags = quote.css("div.tags a.tag::text").extract() + >>> tags = quote.css("div.tags a.tag::text").getall() >>> tags ['change', 'deep-thoughts', 'thinking', 'world'] @@ -405,7 +405,7 @@ quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): ... text = quote.css("span.text::text").get() ... author = quote.css("small.author::text").get() - ... tags = quote.css("div.tags a.tag::text").extract() + ... tags = quote.css("div.tags a.tag::text").getall() ... print(dict(text=text, author=author, tags=tags)) {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} @@ -438,7 +438,7 @@ in the callback, as you can see below:: yield { 'text': quote.css('span.text::text').get(), 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'tags': quote.css('div.tags a.tag::text').getall(), } If you run this spider, it will output the extracted data with the log:: @@ -543,7 +543,7 @@ page, extracting data from it:: yield { 'text': quote.css('span.text::text').get(), 'author': quote.css('small.author::text').get(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'tags': quote.css('div.tags a.tag::text').getall(), } next_page = response.css('li.next a::attr(href)').get() @@ -594,7 +594,7 @@ As a shortcut for creating Request objects you can use yield { 'text': quote.css('span.text::text').get(), 'author': quote.css('span small::text').get(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'tags': quote.css('div.tags a.tag::text').getall(), } next_page = response.css('li.next a::attr(href)').get() diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 9de6abef7..68a0b19b5 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -184,8 +184,8 @@ After that, we can start playing with the objects:: >>> fetch("https://reddit.com") - >>> response.xpath('//title/text()').extract() - ['reddit: the front page of the internet'] + >>> response.xpath('//title/text()').get() + 'reddit: the front page of the internet' >>> request = request.replace(method="POST") diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 4505b7315..a08dc30f2 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback:: ] def parse(self, response): - for h3 in response.xpath('//h3').extract(): + for h3 in response.xpath('//h3').getall(): yield {"title": h3} - for url in response.xpath('//a/@href').extract(): - yield scrapy.Request(url, callback=self.parse) + for href in response.xpath('//a/@href').getall(): + yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; to give data more structure you can use :ref:`topics-items`:: @@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`:: yield scrapy.Request('http://www.example.com/3.html', self.parse) def parse(self, response): - for h3 in response.xpath('//h3').extract(): + for h3 in response.xpath('//h3').getall(): yield MyItem(title=h3) - for url in response.xpath('//a/@href').extract(): - yield scrapy.Request(url, callback=self.parse) + for href in response.xpath('//a/@href').getall(): + yield scrapy.Request(response.urljoin(href), self.parse) .. _spiderargs: @@ -545,7 +545,7 @@ These spiders are pretty easy to use, let's have a look at one example:: itertag = 'item' def parse_node(self, response, node): - self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract())) + self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall())) item = TestItem() item['id'] = node.xpath('@id').get()