From 2c3b2158c99953823500f275ccc5206c11c9a811 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 18 Sep 2018 05:02:17 +0500 Subject: [PATCH] DOC address @stummjr's review comments * fixed several small issues * re-written "Creating Selectors" section * fixed remaining .extract usage in tests --- docs/intro/tutorial.rst | 6 ++-- docs/topics/selectors.rst | 52 +++++++++++++++++++---------------- tests/test_utils_iterators.py | 4 +-- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 46e84b21c..ad17ef096 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -285,9 +285,9 @@ There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail to be scraped, you can at least get **some** data. -Besides the :meth:`~scrapy.selector.Selector.extract` and +Besides the :meth:`~scrapy.selector.SelectorList.getall` and :meth:`~scrapy.selector.SelectorList.get` methods, you can also use -the :meth:`~scrapy.selector.Selector.re` method to extract using `regular +the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular expressions`_:: >>> response.css('title::text').re(r'Quotes.*') @@ -649,7 +649,7 @@ this time for scraping author information:: def parse_author(self, response): def extract_with_css(query): - return response.css(query).get().strip() + return response.css(query).get(default='').strip() yield { 'name': extract_with_css('h3.author-title::text'), diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 20c3fff3c..68913c697 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -50,28 +50,8 @@ Constructing selectors .. highlight:: python -Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class -constructed by passing **text** or :class:`~scrapy.http.TextResponse` -object. It automatically chooses the best parsing rules (XML vs HTML) based on -input type:: - - >>> from scrapy.selector import Selector - >>> from scrapy.http import HtmlResponse - -Constructing from text:: - - >>> body = 'good' - >>> Selector(text=body).xpath('//span/text()').get() - 'good' - -Constructing from response:: - - >>> response = HtmlResponse(url='http://example.com', body=body) - >>> Selector(response=response).xpath('//span/text()').get() - 'good' - -For convenience, response objects expose a selector on `.selector` attribute. -By using it you can ensure the response body is parsed only once:: +Response objects expose a :class:`~scrapy.selector.Selector` instance +on ``.selector`` attribute:: >>> response.selector.xpath('//span/text()').get() 'good' @@ -84,10 +64,34 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:: >>> response.css('span::text').get() 'good' +Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class +constructed by passing either :class:`~scrapy.http.TextResponse` object or +markup as an unicode string (in ``text`` argument). Usually there is no need to construct Scrapy selectors manually: ``response`` object is available in Spider callbacks, so in most cases it is more convenient to use ``response.css()`` and ``response.xpath()`` -shortcuts. +shortcuts. By using ``response.selector`` or one of these shortcuts +you can also ensure the response body is parsed only once. + +But if required, it is possible to use ``Selector`` directly. +Constructing from text:: + + >>> from scrapy.selector import Selector + >>> body = 'good' + >>> Selector(text=body).xpath('//span/text()').get() + 'good' + +Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of +:class:`~scrapy.http.TextResponse` subclasses:: + + >>> from scrapy.selector import Selector + >>> from scrapy.http import HtmlResponse + >>> response = HtmlResponse(url='http://example.com', body=body) + >>> Selector(response=response).xpath('//span/text()').get() + 'good' + +``Selector`` automatically chooses the best parsing rules +(XML vs HTML) based on input type. Using selectors --------------- @@ -139,7 +143,7 @@ is returned. ``.getall()`` returns a list with all results. Notice that CSS selectors can select text or attribute nodes using CSS3 pseudo-elements:: - >>> selector.css('title::text').get() + >>> response.css('title::text').get() 'Example website' As you can see, ``.xpath()`` and ``.css()`` methods return a diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 00eb78068..2d845697e 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -32,8 +32,8 @@ class XmliterTestCase(unittest.TestCase): for x in self.xmliter(response, 'product'): attrs.append(( x.attrib['id'], - x.xpath("name/text()").extract(), - x.xpath("./type/text()").extract())) + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall())) self.assertEqual(attrs, [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])