diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 50d571309..c4165e238 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -129,7 +129,6 @@ For more information about XPath see the `XPath reference`_. Finally, here's the spider code:: - import scrapy from scrapy.contrib.spiders import CrawlSpider, Rule from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor @@ -141,12 +140,11 @@ Finally, here's the spider code:: rules = [Rule(SgmlLinkExtractor(allow=['/tor/\d+']), 'parse_torrent')] def parse_torrent(self, response): - sel = scrapy.Selector(response) torrent = TorrentItem() torrent['url'] = response.url - torrent['name'] = sel.xpath("//h1/text()").extract() - torrent['description'] = sel.xpath("//div[@id='description']").extract() - torrent['size'] = sel.xpath("//div[@id='info-left']/p[2]/text()[2]").extract() + torrent['name'] = response.xpath("//h1/text()").extract() + torrent['description'] = response.xpath("//div[@id='description']").extract() + torrent['size'] = response.xpath("//div[@id='info-left']/p[2]/text()[2]").extract() return torrent The ``TorrentItem`` class is :ref:`defined above `. diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8d1693b28..64b9dca62 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -175,9 +175,9 @@ Scrapy creates :class:`scrapy.Request ` objects for each URL in the ``start_urls`` attribute of the Spider, and assigns them the ``parse`` method of the spider as their callback function. -These Requests are scheduled, then executed, and -:class:`scrapy.http.Response` objects are returned and then fed back to the -spider, through the :meth:`~scrapy.spider.Spider.parse` method. +These Requests are scheduled, then executed, and :class:`scrapy.http.Response` +objects are returned and then fed back to the spider, through the +:meth:`~scrapy.spider.Spider.parse` method. Extracting Items ---------------- @@ -210,9 +210,9 @@ These are just a couple of simple examples of what you can do with XPath, but XPath expressions are indeed much more powerful. To learn more about XPath we recommend `this XPath tutorial `_. -For working with XPaths, Scrapy provides a :class:`~scrapy.selector.Selector` -class, which is instantiated with a :class:`~scrapy.http.HtmlResponse` or -:class:`~scrapy.http.XmlResponse` object as first argument. +For working with XPaths, Scrapy provides :class:`~scrapy.selector.Selector` +class and convenient shortcuts to avoid instantiating selectors yourself +everytime you need to select something from a response. You can see selectors as objects that represent nodes in the document structure. So, the first instantiated selectors are associated with the root @@ -262,7 +262,6 @@ This is what the shell looks like:: [s] item {} [s] request [s] response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> - [s] sel \r\n\r\n>> from scrapy.selector import Selector + >>> from scrapy.http import HtmlResponse - class MySpider(scrapy.Spider): - # ... - def parse(self, response): - sel = scrapy.Selector(response) - # Using XPath query - print sel.xpath('//p') - # Using CSS query - print sel.css('p') - # Nesting queries - print sel.xpath('//div[@foo="bar"]').css('span#bold') +Constructing from text:: + >>> body = 'good' + >>> Selector(text=body).xpath('//span/text()').extract() + [u'good'] + +Constructing from response:: + + >>> response = HtmlResponse(url='http://example.com', body=body) + >>> Selector(response=response).xpath('//span/text()').extract() + [u'good'] + +For convenience, response objects exposes a selector on `.selector` attribute, +it's totally OK to use this shortcut when possible:: + + >>> response.selector.xpath('//span/text()').extract() + [u'good'] + Using selectors --------------- @@ -92,66 +101,80 @@ First, let's open the shell:: scrapy shell http://doc.scrapy.org/en/latest/_static/selectors-sample1.html -Then, after the shell loads, you'll have a selector already instantiated and -ready to use in ``sel`` shell variable. +Then, after the shell loads, you'll have the response available as ``response`` +shell variable, and its attached selector in ``response.selector`` attribute. Since we're dealing with HTML, the selector will automatically use an HTML parser. .. highlight:: python So, by looking at the :ref:`HTML code ` of that -page, let's construct an XPath (using an HTML selector) for selecting the text -inside the title tag:: +page, let's construct an XPath for selecting the text inside the title tag:: - >>> sel.xpath('//title/text()') + >>> response.selector.xpath('//title/text()') [] -As you can see, the ``.xpath()`` method returns an +Querying responses using XPath and CSS is so common that responses includes two +convenient shortcuts: ``response.xpath()`` and ``response.css()``:: + + >>> response.xpath('//title/text()') + [] + >>> response.css('title::text') + [] + +As you can see, ``.xpath()`` and ``.css()`` methods returns an :class:`~scrapy.selector.SelectorList` instance, which is a list of new -selectors. This API can be used quickly for extracting nested data. +selectors. This API can be used quickly for selecting nested data:: -To actually extract the textual data, you must call the selector ``.extract()`` -method, as follows:: - - >>> sel.xpath('//title/text()').extract() - [u'Example website'] - -Notice that CSS selectors can select text or attribute nodes using CSS3 -pseudo-elements:: - - >>> sel.css('title::text').extract() - [u'Example website'] - -Now we're going to get the base URL and some image links:: - - >>> sel.xpath('//base/@href').extract() - [u'http://example.com/'] - - >>> sel.css('base::attr(href)').extract() - [u'http://example.com/'] - - >>> sel.xpath('//a[contains(@href, "image")]/@href').extract() - [u'image1.html', - u'image2.html', - u'image3.html', - u'image4.html', - u'image5.html'] - - >>> sel.css('a[href*=image]::attr(href)').extract() - [u'image1.html', - u'image2.html', - u'image3.html', - u'image4.html', - u'image5.html'] - - >>> sel.xpath('//a[contains(@href, "image")]/img/@src').extract() + >>> response.css('img').xpath('@src').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', u'image4_thumb.jpg', u'image5_thumb.jpg'] - >>> sel.css('a[href*=image] img::attr(src)').extract() +To actually extract the textual data, you must call the selector ``.extract()`` +method, as follows:: + + >>> response.xpath('//title/text()').extract() + [u'Example website'] + +Notice that CSS selectors can select text or attribute nodes using CSS3 +pseudo-elements:: + + >>> response.css('title::text').extract() + [u'Example website'] + +Now we're going to get the base URL and some image links:: + + >>> response.xpath('//base/@href').extract() + [u'http://example.com/'] + + >>> response.css('base::attr(href)').extract() + [u'http://example.com/'] + + >>> response.xpath('//a[contains(@href, "image")]/@href').extract() + [u'image1.html', + u'image2.html', + u'image3.html', + u'image4.html', + u'image5.html'] + + >>> response.css('a[href*=image]::attr(href)').extract() + [u'image1.html', + u'image2.html', + u'image3.html', + u'image4.html', + u'image5.html'] + + >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract() + [u'image1_thumb.jpg', + u'image2_thumb.jpg', + u'image3_thumb.jpg', + u'image4_thumb.jpg', + u'image5_thumb.jpg'] + + >>> response.css('a[href*=image] img::attr(src)').extract() [u'image1_thumb.jpg', u'image2_thumb.jpg', u'image3_thumb.jpg', @@ -167,7 +190,7 @@ The selection methods (``.xpath()`` or ``.css()``) returns a list of selectors of the same type, so you can call the selection methods for those selectors too. Here's an example:: - >>> links = sel.xpath('//a[contains(@href, "image")]') + >>> links = response.xpath('//a[contains(@href, "image")]') >>> links.extract() [u'Name: My image 1
', u'Name: My image 2
', @@ -196,7 +219,7 @@ can't construct nested ``.re()`` calls. Here's an example used to extract images names from the :ref:`HTML code ` above:: - >>> sel.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') + >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') [u'My image 1', u'My image 2', u'My image 3', @@ -215,7 +238,7 @@ with ``/``, that XPath will be absolute to the document and not relative to the For example, suppose you want to extract all ``

`` elements inside ``

`` elements. First, you would get all ``
`` elements:: - >>> divs = sel.xpath('//div') + >>> divs = response.xpath('//div') At first, you may be tempted to use the following approach, which is wrong, as it actually extracts all ``

`` elements from the document, not only those @@ -429,6 +452,10 @@ Built-in Selectors reference ``query`` is a string containing the XPATH query to apply. + .. note:: + + For convenience this method can be called as ``response.xpath()`` + .. method:: css(query) Apply the given CSS selector and return a :class:`SelectorList` instance. @@ -438,6 +465,10 @@ Built-in Selectors reference In the background, CSS queries are translated into XPath queries using `cssselect`_ library and run ``.xpath()`` method. + .. note:: + + For convenience this method can be called as ``response.css()`` + .. method:: extract() Serialize and return the matched nodes as a list of unicode strings. @@ -570,14 +601,14 @@ First, we open the shell with the url we want to scrape:: Once in the shell we can try selecting all ```` objects and see that it doesn't work (because the Atom XML namespace is obfuscating those nodes):: - >>> sel.xpath("//link") + >>> response.xpath("//link") [] But once we call the :meth:`Selector.remove_namespaces` method, all nodes can be accessed directly by their names:: - >>> sel.remove_namespaces() - >>> sel.xpath("//link") + >>> response.selector.remove_namespaces() + >>> response.xpath("//link") [, ... diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 929e21e10..c709f1d39 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -231,11 +231,10 @@ Another example returning multiple Requests and Items from a single callback:: ] def parse(self, response): - sel = scrapy.Selector(response) - for h3 in sel.xpath('//h3').extract(): + for h3 in response.xpath('//h3').extract(): yield MyItem(title=h3) - for url in sel.xpath('//a/@href').extract(): + for url in response.xpath('//a/@href').extract(): yield scrapy.Request(url, callback=self.parse) .. module:: scrapy.contrib.spiders @@ -332,12 +331,10 @@ Let's now take a look at an example CrawlSpider with rules:: def parse_item(self, response): self.log('Hi, this is an item page! %s' % response.url) - - sel = scrapy.Selector(response) item = scrapy.Item() - item['id'] = sel.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = sel.xpath('//td[@id="item_name"]/text()').extract() - item['description'] = sel.xpath('//td[@id="item_description"]/text()').extract() + item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') + item['name'] = response.xpath('//td[@id="item_name"]/text()').extract() + item['description'] = response.xpath('//td[@id="item_description"]/text()').extract() return item diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 618d3970e..14030d8e5 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -19,6 +19,7 @@ class TextResponse(Response): self._encoding = kwargs.pop('encoding', None) self._cached_benc = None self._cached_ubody = None + self._cached_selector = None super(TextResponse, self).__init__(*args, **kwargs) def _set_url(self, url): @@ -88,3 +89,16 @@ class TextResponse(Response): @memoizemethod_noargs def _body_declared_encoding(self): return html_body_declared_encoding(self.body) + + @property + def selector(self): + from scrapy.selector import Selector + if self._cached_selector is None: + self._cached_selector = Selector(self) + return self._cached_selector + + def xpath(self, query): + return self.selector.xpath(query) + + def css(self, query): + return self.selector.css(query) diff --git a/scrapy/shell.py b/scrapy/shell.py index 6d81e0760..323f55f8d 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -6,16 +6,16 @@ See documentation in docs/topics/shell.rst from __future__ import print_function import signal +import warnings from twisted.internet import reactor, threads, defer from twisted.python import threadable from w3lib.url import any_to_uri from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import BaseItem -from scrapy.selector import Selector from scrapy.settings import Settings from scrapy.spider import Spider from scrapy.utils.console import start_python_console @@ -27,7 +27,7 @@ from scrapy.utils.spider import create_spider_for_request class Shell(object): relevant_classes = (Crawler, Spider, Request, Response, BaseItem, - Selector, Settings) + Settings) def __init__(self, crawler, update_vars=None, code=None): self.crawler = crawler @@ -99,7 +99,7 @@ class Shell(object): self.vars['spider'] = spider self.vars['request'] = request self.vars['response'] = response - self.vars['sel'] = Selector(response) + self.vars['sel'] = _SelectorProxy(response) if self.inthread: self.vars['fetch'] = self.fetch self.vars['view'] = open_in_browser @@ -157,3 +157,15 @@ def _request_deferred(request): request.callback, request.errback = d.callback, d.errback return d + + +class _SelectorProxy(object): + + def __init__(self, response): + self._proxiedresponse = response + + def __getattr__(self, name): + warnings.warn('"sel" shortcut is deprecated. Use "response.xpath()", ' + '"response.css()" or "response.selector" instead', + category=ScrapyDeprecationWarning, stacklevel=2) + return getattr(self._proxiedresponse.selector, name) diff --git a/scrapy/templates/spiders/basic.tmpl b/scrapy/templates/spiders/basic.tmpl index da5c57a39..2d5e64fd3 100644 --- a/scrapy/templates/spiders/basic.tmpl +++ b/scrapy/templates/spiders/basic.tmpl @@ -1,5 +1,6 @@ import scrapy + class $classname(scrapy.Spider): name = "$name" allowed_domains = ["$domain"] diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 25e84f940..c0f7f1921 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -3,6 +3,7 @@ from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor from scrapy.contrib.spiders import CrawlSpider, Rule from $project_name.items import ${ProjectName}Item + class $classname(CrawlSpider): name = '$name' allowed_domains = ['$domain'] @@ -13,9 +14,8 @@ class $classname(CrawlSpider): ) def parse_item(self, response): - sel = scrapy.Selector(response) i = ${ProjectName}Item() - #i['domain_id'] = sel.xpath('//input[@id="sid"]/@value').extract() - #i['name'] = sel.xpath('//div[@id="name"]').extract() - #i['description'] = sel.xpath('//div[@id="description"]').extract() + #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() + #i['name'] = response.xpath('//div[@id="name"]').extract() + #i['description'] = response.xpath('//div[@id="description"]').extract() return i diff --git a/scrapy/templates/spiders/csvfeed.tmpl b/scrapy/templates/spiders/csvfeed.tmpl index 7551686a6..6ef84b524 100644 --- a/scrapy/templates/spiders/csvfeed.tmpl +++ b/scrapy/templates/spiders/csvfeed.tmpl @@ -1,6 +1,7 @@ from scrapy.contrib.spiders import CSVFeedSpider from $project_name.items import ${ProjectName}Item + class $classname(CSVFeedSpider): name = '$name' allowed_domains = ['$domain'] diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 915affee5..ffff70632 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -1,6 +1,7 @@ from scrapy.contrib.spiders import XMLFeedSpider from $project_name.items import ${ProjectName}Item + class $classname(XMLFeedSpider): name = '$name' allowed_domains = ['$domain'] diff --git a/scrapy/tests/test_command_shell.py b/scrapy/tests/test_command_shell.py index 1fbb4b28a..a56236d54 100644 --- a/scrapy/tests/test_command_shell.py +++ b/scrapy/tests/test_command_shell.py @@ -31,7 +31,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'sel.xpath("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), 'Works') diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index 0809340a1..26a628182 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -2,6 +2,7 @@ import unittest from w3lib.encoding import resolve_encoding from scrapy.http import Request, Response, TextResponse, HtmlResponse, XmlResponse, Headers +from scrapy.selector import Selector class BaseResponseTest(unittest.TestCase): @@ -112,6 +113,7 @@ class BaseResponseTest(unittest.TestCase): self.assertRaises(AttributeError, setattr, r, 'url', 'http://example2.com') self.assertRaises(AttributeError, setattr, r, 'body', 'xxx') + class ResponseText(BaseResponseTest): def test_no_unicode_url(self): @@ -258,13 +260,48 @@ class TextResponseTest(BaseResponseTest): #r = self.response_class("http://www.example.com", body='PREFIX\xe3\xabSUFFIX') #assert u'\ufffd' in r.body_as_unicode(), repr(r.body_as_unicode()) + def test_selector(self): + body = "Some page" + response = self.response_class("http://www.example.com", body=body) + + self.assertIsInstance(response.selector, Selector) + self.assertEqual(response.selector.type, 'html') + self.assertIs(response.selector, response.selector) # property is cached + self.assertIs(response.selector.response, response) + + self.assertEqual( + response.selector.xpath("//title/text()").extract(), + [u'Some page'] + ) + self.assertEqual( + response.selector.css("title::text").extract(), + [u'Some page'] + ) + self.assertEqual( + response.selector.re("Some (.*)"), + [u'page'] + ) + + def test_selector_shortcuts(self): + body = "Some page" + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("//title/text()").extract(), + response.selector.xpath("//title/text()").extract(), + ) + self.assertEqual( + response.css("title::text").extract(), + response.selector.css("title::text").extract(), + ) + class HtmlResponseTest(TextResponseTest): response_class = HtmlResponse def test_html_encoding(self): - + body = """Some page Price: \xa3100' """ @@ -328,6 +365,30 @@ class XmlResponseTest(TextResponseTest): self._assert_response_values(r6, 'iso-8859-1', body2) self._assert_response_values(r7, 'utf-8', body2) + def test_selector(self): + body = 'value' + response = self.response_class("http://www.example.com", body=body) + + self.assertIsInstance(response.selector, Selector) + self.assertEqual(response.selector.type, 'xml') + self.assertIs(response.selector, response.selector) # property is cached + self.assertIs(response.selector.response, response) + + self.assertEqual( + response.selector.xpath("//elem/text()").extract(), + [u'value'] + ) + + def test_selector_shortcuts(self): + body = 'value' + response = self.response_class("http://www.example.com", body=body) + + self.assertEqual( + response.xpath("//elem/text()").extract(), + response.selector.xpath("//elem/text()").extract(), + ) + + if __name__ == "__main__": unittest.main()