diff --git a/.gitignore b/.gitignore index 406146e5f..7392ed31e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,10 @@ dist .idea htmlcov/ .coverage +.pytest_cache/ .coverage.* .cache/ +.pytest_cache/ # Windows Thumbs.db diff --git a/.travis.yml b/.travis.yml index 4218d13bf..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: python -sudo: false branches: only: - master diff --git a/MANIFEST.in b/MANIFEST.in index 94de4f3bf..ae7db51fa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,13 +3,24 @@ include AUTHORS include INSTALL include LICENSE include MANIFEST.in +include NEWS + include scrapy/VERSION include scrapy/mime.types + +include codecov.yml +include conftest.py +include pytest.ini +include requirements-*.txt +include tox.ini + recursive-include scrapy/templates * recursive-include scrapy license.txt recursive-include docs * prune docs/build + recursive-include extras * recursive-include bin * recursive-include tests * + global-exclude __pycache__ *.py[cod] diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4a9aa3cfb..daec7fcb7 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. -For more detailed and platform specifics instructions, read on. +For more detailed and platform specifics instructions, as well as +troubleshooting information, read on. Things that are good to know @@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency. To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +.. _intro-install-troubleshooting: + +Troubleshooting +=============== + +AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' +---------------------------------------------------------------- + +After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an +exception with the following traceback:: + + […] + File "[…]/site-packages/twisted/protocols/tls.py", line 63, in + from twisted.internet._sslverify import _setAcceptableProtocols + File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in + TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1, + AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' + +The reason you get this exception is that your system or virtual environment +has a version of pyOpenSSL that your version of Twisted does not support. + +To install a version of pyOpenSSL that your version of Twisted supports, +reinstall Twisted with the :code:`tls` extra option:: + + pip install twisted[tls] + +For details, see `Issue #2473 `_. + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 6f1c2c43f..9d7c94d39 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.xpath('span/small/text()').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.xpath('span/small/text()').get(), } - next_page = response.css('li.next a::attr("href")').extract_first() + next_page = response.css('li.next a::attr("href")').get() if next_page is not None: yield response.follow(next_page, self.parse) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fa6dc274d..41e61542a 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -22,9 +22,7 @@ Scrapy is written in Python_. If you're new to the language you might want to start by getting an idea of what the language is like, to get the most out of Scrapy. -If you're already familiar with other languages, and want to learn Python -quickly, we recommend reading through `Dive Into Python 3`_. Alternatively, -you can follow the `Python Tutorial`_. +If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource. If you're new to programming and want to start with Python, the following books may be useful to you: @@ -40,7 +38,6 @@ as well as the `suggested resources in the learnpython-subreddit`_. .. _Python: https://www.python.org/ .. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers -.. _Dive Into Python 3: http://www.diveintopython3.net .. _Python Tutorial: https://docs.python.org/3/tutorial .. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/ .. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/ @@ -254,7 +251,7 @@ data. To extract the text from the title above, you can do:: - >>> response.css('title::text').extract() + >>> response.css('title::text').getall() ['Quotes to Scrape'] There are two things to note here: one is that we've added ``::text`` to the @@ -262,32 +259,33 @@ CSS query, to mean we want to select only the text elements directly inside ```` element. If we don't specify ``::text``, we'd get the full title element, including its tags:: - >>> response.css('title').extract() + >>> response.css('title').getall() ['<title>Quotes to Scrape'] -The other thing is that the result of calling ``.extract()`` is a list, because -we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When -you know you just want the first result, as in this case, you can do:: +The other thing is that the result of calling ``.getall()`` is a list: it is +possible that a selector returns more than one result, so we extract them all. +When you know you just want the first result, as in this case, you can do:: - >>> response.css('title::text').extract_first() + >>> response.css('title::text').get() 'Quotes to Scrape' As an alternative, you could've written:: - >>> response.css('title::text')[0].extract() + >>> response.css('title::text')[0].get() 'Quotes to Scrape' -However, using ``.extract_first()`` avoids an ``IndexError`` and returns -``None`` when it doesn't find any element matching the selection. +However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList` +instance avoids an ``IndexError`` and returns ``None`` when it doesn't +find any element matching the selection. There's a lesson here: for most scraping code, you want it to be resilient to errors due to things not being found on a page, so that even if some parts fail to be scraped, you can at least get **some** data. -Besides the :meth:`~scrapy.selector.Selector.extract` and -:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use -the :meth:`~scrapy.selector.Selector.re` method to extract using `regular -expressions`:: +Besides the :meth:`~scrapy.selector.SelectorList.getall` and +:meth:`~scrapy.selector.SelectorList.get` methods, you can also use +the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular +expressions`_:: >>> response.css('title::text').re(r'Quotes.*') ['Quotes to Scrape'] @@ -298,7 +296,8 @@ expressions`:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools (see section about :ref:`topics-developer-tools`). +You can use your browser developer tools to inspect the HTML and come up +with a selector (see section about :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. @@ -314,7 +313,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:: >>> response.xpath('//title') [] - >>> response.xpath('//title/text()').extract_first() + >>> response.xpath('//title/text()').get() 'Quotes to Scrape' XPath expressions are very powerful, and are the foundation of Scrapy @@ -383,17 +382,17 @@ variable, so that we can run our CSS selectors directly on a particular quote:: Now, let's extract ``title``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").extract_first() + >>> title = quote.css("span.text::text").get() >>> title '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' - >>> author = quote.css("small.author::text").extract_first() + >>> author = quote.css("small.author::text").get() >>> author 'Albert Einstein' -Given that the tags are a list of strings, we can use the ``.extract()`` method +Given that the tags are a list of strings, we can use the ``.getall()`` method to get all of them:: - >>> tags = quote.css("div.tags a.tag::text").extract() + >>> tags = quote.css("div.tags a.tag::text").getall() >>> tags ['change', 'deep-thoughts', 'thinking', 'world'] @@ -401,9 +400,9 @@ Having figured out how to extract each bit, we can now iterate over all the quotes elements and put them together into a Python dictionary:: >>> for quote in response.css("div.quote"): - ... text = quote.css("span.text::text").extract_first() - ... author = quote.css("small.author::text").extract_first() - ... tags = quote.css("div.tags a.tag::text").extract() + ... text = quote.css("span.text::text").get() + ... author = quote.css("small.author::text").get() + ... tags = quote.css("div.tags a.tag::text").getall() ... print(dict(text=text, author=author, tags=tags)) {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'} {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'} @@ -434,9 +433,9 @@ in the callback, as you can see below:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), } If you run this spider, it will output the extracted data with the log:: @@ -508,16 +507,22 @@ markup: We can try extracting it in the shell:: - >>> response.css('li.next a').extract_first() + >>> response.css('li.next a').get() 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, Scrapy supports a CSS extension that let's you select the attribute contents, like this:: - >>> response.css('li.next a::attr(href)').extract_first() + >>> response.css('li.next a::attr(href)').get() '/page/2/' +There is also an ``attrib`` property available +(see :ref:`selecting-attributes` for more):: + + >>> response.css('li.next a').attrib['href'] + '/page/2' + Let's see now our spider modified to recursively follow the link to the next page, extracting data from it:: @@ -533,12 +538,12 @@ page, extracting data from it:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: next_page = response.urljoin(next_page) yield scrapy.Request(next_page, callback=self.parse) @@ -584,12 +589,12 @@ As a shortcut for creating Request objects you can use def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('span small::text').extract_first(), - 'tags': quote.css('div.tags a.tag::text').extract(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('span small::text').get(), + 'tags': quote.css('div.tags a.tag::text').getall(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: yield response.follow(next_page, callback=self.parse) @@ -641,7 +646,7 @@ this time for scraping author information:: def parse_author(self, response): def extract_with_css(query): - return response.css(query).extract_first().strip() + return response.css(query).get(default='').strip() yield { 'name': extract_with_css('h3.author-title::text'), @@ -710,11 +715,11 @@ with a specific tag, building the URL based on the argument:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').extract_first(), - 'author': quote.css('small.author::text').extract_first(), + 'text': quote.css('span.text::text').get(), + 'author': quote.css('small.author::text').get(), } - next_page = response.css('li.next a::attr(href)').extract_first() + next_page = response.css('li.next a::attr(href)').get() if next_page is not None: yield response.follow(next_page, self.parse) @@ -738,4 +743,3 @@ modeling the scraped data. If you prefer to play with an example project, check the :ref:`intro-examples` section. .. _JSON: https://en.wikipedia.org/wiki/JSON -.. _dirbot: https://github.com/scrapy/dirbot diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 3088017cb..97f8311de 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`) -* ``SCRAPY_PROJECT`` +* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`) * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) .. _topics-project-structure: @@ -71,6 +71,33 @@ the project settings. Here is an example:: [settings] default = myproject.settings +.. _topics-project-envvar: + +Sharing the root directory between projects +=========================================== + +A project root directory, the one that contains the ``scrapy.cfg``, may be +shared by multiple Scrapy projects, each with its own settings module. + +In that case, you must define one or more aliases for those settings modules +under ``[settings]`` in your ``scrapy.cfg`` file:: + + [settings] + default = myproject1.settings + project1 = myproject1.settings + project2 = myproject2.settings + +By default, the ``scrapy`` command-line tool will use the ``default`` settings. +Use the ``SCRAPY_PROJECT`` environment variable to specify a different project +for ``scrapy`` to use:: + + $ scrapy settings --get BOT_NAME + Project 1 Bot + $ export SCRAPY_PROJECT=project2 + $ scrapy settings --get BOT_NAME + Project 2 Bot + + Using the ``scrapy`` tool ========================= @@ -458,9 +485,9 @@ Usage example:: >>> STATUS DEPTH LEVEL 1 <<< # Scraped Items ------------------------------------------------------------ - [{'name': u'Example item', - 'category': u'Furniture', - 'length': u'12 cm'}] + [{'name': 'Example item', + 'category': 'Furniture', + 'length': '12 cm'}] # Requests ----------------------------------------------------------------- [] diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index ba1421c42..70f20d4ed 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -86,8 +86,11 @@ override three methods: .. method:: Contract.adjust_request_args(args) This receives a ``dict`` as an argument containing default arguments - for :class:`~scrapy.http.Request` object. Must return the same or a - modified version of it. + for request object. :class:`~scrapy.http.Request` is used by default, + but this can be changed with the ``request_cls`` attribute. + If multiple contracts in chain have this attribute defined, the last one is used. + + Must return the same or a modified version of it. .. method:: Contract.pre_process(response) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index d1991c02f..f93aa2c72 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -18,11 +18,13 @@ Consider the following scrapy spider below:: ) def parse(self, response): - # collect `item_urls` + # + # collect `item_urls` for item_url in item_urls: yield scrapy.Request(item_url, self.parse_item) def parse_item(self, response): + # item = MyItem() # populate `item` fields # and extract item_details_url diff --git a/docs/topics/items.rst b/docs/topics/items.rst index 4423bbda2..ae44aecd3 100644 --- a/docs/topics/items.rst +++ b/docs/topics/items.rst @@ -86,7 +86,7 @@ Creating items :: >>> product = Product(name='Desktop PC', price=1000) - >>> print product + >>> print(product) Product(name='Desktop PC', price=1000) Getting field values @@ -161,11 +161,11 @@ Other common tasks Copying items:: >>> product2 = Product(product) - >>> print product2 + >>> print(product2) Product(name='Desktop PC', price=1000) >>> product3 = product2.copy() - >>> print product3 + >>> print(product3) Product(name='Desktop PC', price=1000) Creating dicts from items:: diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst index 06c7fff3d..8e1574376 100644 --- a/docs/topics/jobs.rst +++ b/docs/topics/jobs.rst @@ -84,7 +84,7 @@ So, for example, this won't work:: return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg)) def other_callback(self, response, somearg): - print "the argument passed is:", somearg + print("the argument passed is: %s" % somearg) But this will:: @@ -94,7 +94,7 @@ But this will:: def other_callback(self, response): somearg = response.meta['somearg'] - print "the argument passed is:", somearg + print("the argument passed is: %s" % somearg) If you wish to log the requests that couldn't be serialized, you can set the :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page. diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index a895b535c..f3b6aa4a1 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -678,10 +678,10 @@ Here is a list of all built-in processors: >>> from scrapy.loader.processors import Join >>> proc = Join() >>> proc(['one', 'two', 'three']) - u'one two three' + 'one two three' >>> proc = Join('
') >>> proc(['one', 'two', 'three']) - u'one
two
three' + 'one
two
three' .. class:: Compose(\*functions, \**default_loader_context) @@ -744,9 +744,9 @@ Here is a list of all built-in processors: ... return None if x == 'world' else x ... >>> from scrapy.loader.processors import MapCompose - >>> proc = MapCompose(filter_world, unicode.upper) - >>> proc([u'hello', u'world', u'this', u'is', u'scrapy']) - [u'HELLO, u'THIS', u'IS', u'SCRAPY'] + >>> proc = MapCompose(filter_world, str.upper) + >>> proc(['hello', 'world', 'this', 'is', 'scrapy']) + ['HELLO, 'THIS', 'IS', 'SCRAPY'] As with the Compose processor, functions can receive Loader contexts, and constructor keyword arguments are used as default context values. See @@ -772,7 +772,7 @@ Here is a list of all built-in processors: >>> import json >>> proc_single_json_str = Compose(json.loads, SelectJmes("foo")) >>> proc_single_json_str('{"foo": "bar"}') - u'bar' + 'bar' >>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo'))) >>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]') - [u'bar'] + ['bar'] diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst index 8ac40c3cc..9dced7473 100644 --- a/docs/topics/selectors.rst +++ b/docs/topics/selectors.rst @@ -6,7 +6,7 @@ Selectors When you're scraping web pages, the most common task you need to perform is to extract data from the HTML source. There are several libraries available to -achieve this: +achieve this, such as: * `BeautifulSoup`_ is a very popular web scraping library among Python programmers which constructs a Python object based on the structure of the @@ -25,16 +25,14 @@ either by `XPath`_ or `CSS`_ expressions. used with HTML. `CSS`_ is a language for applying styles to HTML documents. It defines selectors to associate those styles with specific HTML elements. -Scrapy selectors are built over the `lxml`_ library, which means they're very -similar in speed and parsing accuracy. +.. note:: + Scrapy Selectors is a thin wrapper around `parsel`_ library; the purpose of + this wrapper is to provide better integration with Scrapy Response objects. -This page explains how selectors work and describes their API which is very -small and simple, unlike the `lxml`_ API which is much bigger because the -`lxml`_ library can be used for many other tasks, besides selecting markup -documents. - -For a complete reference of the selectors API see -:ref:`Selector reference ` + `parsel`_ is a stand-alone web scraping library which can be used without + Scrapy. It uses `lxml`_ library under the hood, and implements an + easy API on top of lxml API. It means Scrapy selectors are very similar + in speed and parsing accuracy to lxml. .. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/ .. _lxml: http://lxml.de/ @@ -42,7 +40,7 @@ For a complete reference of the selectors API see .. _cssselect: https://pypi.python.org/pypi/cssselect/ .. _XPath: https://www.w3.org/TR/xpath .. _CSS: https://www.w3.org/TR/selectors - +.. _parsel: https://parsel.readthedocs.io/ Using selectors =============== @@ -52,32 +50,48 @@ Constructing selectors .. highlight:: python +Response objects expose a :class:`~scrapy.selector.Selector` instance +on ``.selector`` attribute:: + + >>> response.selector.xpath('//span/text()').get() + 'good' + +Querying responses using XPath and CSS is so common that responses include two +more shortcuts: ``response.xpath()`` and ``response.css()``:: + + >>> response.xpath('//span/text()').get() + 'good' + >>> response.css('span::text').get() + 'good' + Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class -constructed by passing **text** or :class:`~scrapy.http.TextResponse` -object. It automatically chooses the best parsing rules (XML vs HTML) based on -input type:: +constructed by passing either :class:`~scrapy.http.TextResponse` object or +markup as an unicode string (in ``text`` argument). +Usually there is no need to construct Scrapy selectors manually: +``response`` object is available in Spider callbacks, so in most cases +it is more convenient to use ``response.css()`` and ``response.xpath()`` +shortcuts. By using ``response.selector`` or one of these shortcuts +you can also ensure the response body is parsed only once. + +But if required, it is possible to use ``Selector`` directly. +Constructing from text:: + + >>> from scrapy.selector import Selector + >>> body = 'good' + >>> Selector(text=body).xpath('//span/text()').get() + 'good' + +Constructing from response - :class:`~scrapy.http.HtmlResponse` is one of +:class:`~scrapy.http.TextResponse` subclasses:: >>> from scrapy.selector import Selector >>> from scrapy.http import HtmlResponse - -Constructing from text:: - - >>> body = 'good' - >>> Selector(text=body).xpath('//span/text()').extract() - [u'good'] - -Constructing from response:: - >>> response = HtmlResponse(url='http://example.com', body=body) - >>> Selector(response=response).xpath('//span/text()').extract() - [u'good'] - -For convenience, response objects expose a selector on `.selector` attribute, -it's totally OK to use this shortcut when possible:: - - >>> response.selector.xpath('//span/text()').extract() - [u'good'] + >>> Selector(response=response).xpath('//span/text()').get() + 'good' +``Selector`` automatically chooses the best parsing rules +(XML vs HTML) based on input type. Using selectors --------------- @@ -90,7 +104,7 @@ documentation server: .. _topics-selectors-htmlcode: -Here's its HTML code: +For the sake of completeness, here's its full HTML code: .. literalinclude:: ../_static/selectors-sample1.html :language: html @@ -111,90 +125,191 @@ Since we're dealing with HTML, the selector will automatically use an HTML parse So, by looking at the :ref:`HTML code ` of that page, let's construct an XPath for selecting the text inside the title tag:: - >>> response.selector.xpath('//title/text()') - [] - -Querying responses using XPath and CSS is so common that responses include two -convenience shortcuts: ``response.xpath()`` and ``response.css()``:: - >>> response.xpath('//title/text()') - [] - >>> response.css('title::text') - [] + [] + +To actually extract the textual data, you must call the selector ``.get()`` +or ``.getall()`` methods, as follows:: + + >>> response.xpath('//title/text()').getall() + ['Example website'] + >>> response.xpath('//title/text()').get() + 'Example website' + +``.get()`` always returns a single result; if there are several matches, +content of a first match is returned; if there are no matches, None +is returned. ``.getall()`` returns a list with all results. + +Notice that CSS selectors can select text or attribute nodes using CSS3 +pseudo-elements:: + + >>> response.css('title::text').get() + 'Example website' As you can see, ``.xpath()`` and ``.css()`` methods return a :class:`~scrapy.selector.SelectorList` instance, which is a list of new selectors. This API can be used for quickly selecting nested data:: - >>> response.css('img').xpath('@src').extract() - [u'image1_thumb.jpg', - u'image2_thumb.jpg', - u'image3_thumb.jpg', - u'image4_thumb.jpg', - u'image5_thumb.jpg'] + >>> response.css('img').xpath('@src').getall() + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] -To actually extract the textual data, you must call the selector ``.extract()`` -method, as follows:: +If you want to extract only the first matched element, you can call the +selector ``.get()`` (or its alias ``.extract_first()`` commonly used in +previous Scrapy versions):: - >>> response.xpath('//title/text()').extract() - [u'Example website'] + >>> response.xpath('//div[@id="images"]/a/text()').get() + 'Name: My image 1 ' -If you want to extract only first matched element, you can call the selector ``.extract_first()`` +It returns ``None`` if no element was found:: - >>> response.xpath('//div[@id="images"]/a/text()').extract_first() - u'Name: My image 1 ' - -It returns ``None`` if no element was found: - - >>> response.xpath('//div[@id="not-exists"]/text()').extract_first() is None + >>> response.xpath('//div[@id="not-exists"]/text()').get() is None True -A default return value can be provided as an argument, to be used instead of ``None``: +A default return value can be provided as an argument, to be used instead +of ``None``: - >>> response.xpath('//div[@id="not-exists"]/text()').extract_first(default='not-found') + >>> response.xpath('//div[@id="not-exists"]/text()').get(default='not-found') 'not-found' -Notice that CSS selectors can select text or attribute nodes using CSS3 -pseudo-elements:: +Instead of using e.g. ``'@src'`` XPath it is possible to query for attributes +using ``.attrib`` property of a :class:`~scrapy.selector.Selector`:: - >>> response.css('title::text').extract() - [u'Example website'] + >>> [img.attrib['src'] for img in response.css('img')] + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] + +As a shortcut, ``.attrib`` is also available on SelectorList directly; +it returns attributes for the first matching element:: + + >>> response.css('img').attrib['src'] + 'image1_thumb.jpg' + +This is most useful when only a single result is expected, e.g. when selecting +by id, or selecting unique elements on a web page:: + + >>> response.css('base').attrib['href'] + 'http://example.com/' Now we're going to get the base URL and some image links:: - >>> response.xpath('//base/@href').extract() - [u'http://example.com/'] + >>> response.xpath('//base/@href').get() + 'http://example.com/' - >>> response.css('base::attr(href)').extract() - [u'http://example.com/'] + >>> response.css('base::attr(href)').get() + 'http://example.com/' - >>> response.xpath('//a[contains(@href, "image")]/@href').extract() - [u'image1.html', - u'image2.html', - u'image3.html', - u'image4.html', - u'image5.html'] + >>> response.css('base').attrib['href'] + 'http://example.com/' - >>> response.css('a[href*=image]::attr(href)').extract() - [u'image1.html', - u'image2.html', - u'image3.html', - u'image4.html', - u'image5.html'] + >>> response.xpath('//a[contains(@href, "image")]/@href').getall() + ['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] - >>> response.xpath('//a[contains(@href, "image")]/img/@src').extract() - [u'image1_thumb.jpg', - u'image2_thumb.jpg', - u'image3_thumb.jpg', - u'image4_thumb.jpg', - u'image5_thumb.jpg'] + >>> response.css('a[href*=image]::attr(href)').getall() + ['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] - >>> response.css('a[href*=image] img::attr(src)').extract() - [u'image1_thumb.jpg', - u'image2_thumb.jpg', - u'image3_thumb.jpg', - u'image4_thumb.jpg', - u'image5_thumb.jpg'] + >>> response.xpath('//a[contains(@href, "image")]/img/@src').getall() + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] + + >>> response.css('a[href*=image] img::attr(src)').getall() + ['image1_thumb.jpg', + 'image2_thumb.jpg', + 'image3_thumb.jpg', + 'image4_thumb.jpg', + 'image5_thumb.jpg'] + +.. _topics-selectors-css-extensions: + +Extensions to CSS Selectors +--------------------------- + +Per W3C standards, `CSS selectors`_ do not support selecting text nodes +or attribute values. +But selecting these is so essential in a web scraping context +that Scrapy (parsel) implements a couple of **non-standard pseudo-elements**: + +* to select text nodes, use ``::text`` +* to select attribute values, use ``::attr(name)`` where *name* is the + name of the attribute that you want the value of + +.. warning:: + These pseudo-elements are Scrapy-/Parsel-specific. + They will most probably not work with other libraries like + `lxml`_ or `PyQuery`_. + +.. _PyQuery: https://pypi.python.org/pypi/pyquery + +Examples: + +* ``title::text`` selects children text nodes of a descendant ```` element:: + + >>> response.css('title::text').get() + 'Example website' + +* ``*::text`` selects all descendant text nodes of the current selector context:: + + >>> response.css('#images *::text').getall() + ['\n ', + 'Name: My image 1 ', + '\n ', + 'Name: My image 2 ', + '\n ', + 'Name: My image 3 ', + '\n ', + 'Name: My image 4 ', + '\n ', + 'Name: My image 5 ', + '\n '] + +* ``foo::text`` returns no results if ``foo`` element exists, but contains + no text (i.e. text is empty):: + + >>> response.css('img::text').getall() + [] + + This means ``.css('foo::text').get()`` could return None even if an element + exists. Use ``default=''`` if you always want a string:: + + >>> response.css('img::text').get() + >>> response.css('img::text').get(default='') + '' + +* ``a::attr(href)`` selects the *href* attribute value of descendant links:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', + 'image2.html', + 'image3.html', + 'image4.html', + 'image5.html'] + +.. note:: + See also: :ref:`selecting-attributes`. + +.. note:: + You cannot chain these pseudo-elements. But in practice it would not + make much sense: text nodes do not have attributes, and attribute values + are string values already and do not have children nodes. + +.. _CSS Selectors: https://www.w3.org/TR/css3-selectors/#selectors .. _topics-selectors-nesting-selectors: @@ -206,22 +321,65 @@ of the same type, so you can call the selection methods for those selectors too. Here's an example:: >>> links = response.xpath('//a[contains(@href, "image")]') - >>> links.extract() - [u'<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>', - u'<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>', - u'<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>', - u'<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>', - u'<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] + >>> links.getall() + ['<a href="image1.html">Name: My image 1 <br><img src="image1_thumb.jpg"></a>', + '<a href="image2.html">Name: My image 2 <br><img src="image2_thumb.jpg"></a>', + '<a href="image3.html">Name: My image 3 <br><img src="image3_thumb.jpg"></a>', + '<a href="image4.html">Name: My image 4 <br><img src="image4_thumb.jpg"></a>', + '<a href="image5.html">Name: My image 5 <br><img src="image5_thumb.jpg"></a>'] >>> for index, link in enumerate(links): - ... args = (index, link.xpath('@href').extract(), link.xpath('img/@src').extract()) - ... print 'Link number %d points to url %s and image %s' % args + ... args = (index, link.xpath('@href').get(), link.xpath('img/@src').get()) + ... print('Link number %d points to url %r and image %r' % args) - Link number 0 points to url [u'image1.html'] and image [u'image1_thumb.jpg'] - Link number 1 points to url [u'image2.html'] and image [u'image2_thumb.jpg'] - Link number 2 points to url [u'image3.html'] and image [u'image3_thumb.jpg'] - Link number 3 points to url [u'image4.html'] and image [u'image4_thumb.jpg'] - Link number 4 points to url [u'image5.html'] and image [u'image5_thumb.jpg'] + Link number 0 points to url 'image1.html' and image 'image1_thumb.jpg' + Link number 1 points to url 'image2.html' and image 'image2_thumb.jpg' + Link number 2 points to url 'image3.html' and image 'image3_thumb.jpg' + Link number 3 points to url 'image4.html' and image 'image4_thumb.jpg' + Link number 4 points to url 'image5.html' and image 'image5_thumb.jpg' + +.. _selecting-attributes: + +Selecting element attributes +---------------------------- + +There are several ways to get a value of an attribute. First, one can use +XPath syntax:: + + >>> response.xpath("//a/@href").getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +XPath syntax has a few advantages: it is a standard XPath feature, and +``@attributes`` can be used in other parts of an XPath expression - e.g. +it is possible to filter by attribute value. + +Scrapy also provides an extension to CSS selectors (``::attr(...)``) +which allows to get attribute values:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +In addition to that, there is a ``.attrib`` property of Selector. +You can use it if you prefer to lookup attributes in Python +code, without using XPaths or CSS extensions:: + + >>> [a.attrib['href'] for a in response.css('a')] + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +This property is also available on SelectorList; it returns a dictionary +with attributes of a first matching element. It is convenient to use when +a selector is expected to give a single result (e.g. when selecting by element +ID, or when selecting an unique element on a page):: + + >>> response.css('base').attrib + {'href': 'http://example.com/'} + >>> response.css('base').attrib['href'] + 'http://example.com/' + +``.attrib`` property of an empty SelectorList is empty:: + + >>> response.css('foo').attrib + {} Using selectors with regular expressions ---------------------------------------- @@ -235,17 +393,83 @@ Here's an example used to extract image names from the :ref:`HTML code <topics-selectors-htmlcode>` above:: >>> response.xpath('//a[contains(@href, "image")]/text()').re(r'Name:\s*(.*)') - [u'My image 1', - u'My image 2', - u'My image 3', - u'My image 4', - u'My image 5'] + ['My image 1', + 'My image 2', + 'My image 3', + 'My image 4', + 'My image 5'] -There's an additional helper reciprocating ``.extract_first()`` for ``.re()``, -named ``.re_first()``. Use it to extract just the first matching string:: +There's an additional helper reciprocating ``.get()`` (and its +alias ``.extract_first()``) for ``.re()``, named ``.re_first()``. +Use it to extract just the first matching string:: >>> response.xpath('//a[contains(@href, "image")]/text()').re_first(r'Name:\s*(.*)') - u'My image 1' + 'My image 1' + +.. _old-extraction-api: + +extract() and extract_first() +----------------------------- + +If you're a long-time Scrapy user, you're probably familiar +with ``.extract()`` and ``.extract_first()`` selector methods. Many blog posts +and tutorials are using them as well. These methods are still supported +by Scrapy, there are **no plans** to deprecate them. + +However, Scrapy usage docs are now written using ``.get()`` and +``.getall()`` methods. We feel that these new methods result in a more concise +and readable code. + +The following examples show how these methods map to each other. + +1. ``SelectorList.get()`` is the same as ``SelectorList.extract_first()``:: + + >>> response.css('a::attr(href)').get() + 'image1.html' + >>> response.css('a::attr(href)').extract_first() + 'image1.html' + +2. ``SelectorList.getall()`` is the same as ``SelectorList.extract()``:: + + >>> response.css('a::attr(href)').getall() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + >>> response.css('a::attr(href)').extract() + ['image1.html', 'image2.html', 'image3.html', 'image4.html', 'image5.html'] + +2. ``Selector.get()`` is the same as ``Selector.extract()``:: + + >>> response.css('a::attr(href)')[0].get() + 'image1.html' + >>> response.css('a::attr(href)')[0].extract() + 'image1.html' + +4. For consistency, there is also ``Selector.getall()``, which returns a list:: + + >>> response.css('a::attr(href)')[0].getall() + ['image1.html'] + +So, the main difference is that output of ``.get()`` and ``.getall()`` methods +is more predictable: ``.get()`` always returns a single result, ``.getall()`` +always returns a list of all extracted results. With ``.extract()`` method +it was not always obvious if a result is a list or not; to get a single +result either ``.extract()`` or ``.extract_first()`` should be called. + + +.. _topics-selectors-xpaths: + +Working with XPaths +=================== + +Here are some tips which may help you to use XPath with Scrapy selectors +effectively. If you are not much familiar with XPath yet, +you may want to take a look first at this `XPath tutorial`_. + +.. note:: + Some of the tips are based on `this post from ScrapingHub's blog`_. + +.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html +.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ + .. _topics-selectors-relative-xpaths: @@ -266,23 +490,131 @@ it actually extracts all ``<p>`` elements from the document, not only those inside ``<div>`` elements:: >>> for p in divs.xpath('//p'): # this is wrong - gets all <p> from the whole document - ... print p.extract() + ... print(p.get()) This is the proper way to do it (note the dot prefixing the ``.//p`` XPath):: >>> for p in divs.xpath('.//p'): # extracts all <p> inside - ... print p.extract() + ... print(p.get()) Another common case would be to extract all direct ``<p>`` children:: >>> for p in divs.xpath('p'): - ... print p.extract() + ... print(p.get()) For more details about relative XPaths see the `Location Paths`_ section in the XPath specification. .. _Location Paths: https://www.w3.org/TR/xpath#location-paths +When querying by class, consider using CSS +------------------------------------------ + +Because an element can contain multiple CSS classes, the XPath way to select elements +by class is the rather verbose:: + + *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')] + +If you use ``@class='someclass'`` you may end up missing elements that have +other classes, and if you just use ``contains(@class, 'someclass')`` to make up +for that you may end up with more elements that you want, if they have a different +class name that shares the string ``someclass``. + +As it turns out, Scrapy selectors allow you to chain selectors, so most of the time +you can just select by class using CSS and then switch to XPath when needed:: + + >>> from scrapy import Selector + >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>') + >>> sel.css('.shout').xpath('./time/@datetime').getall() + ['2014-07-23 19:00'] + +This is cleaner than using the verbose XPath trick shown above. Just remember +to use the ``.`` in the XPath expressions that will follow. + +Beware of the difference between //node[1] and (//node)[1] +---------------------------------------------------------- + +``//node[1]`` selects all the nodes occurring first under their respective parents. + +``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them. + +Example:: + + >>> from scrapy import Selector + >>> sel = Selector(text=""" + ....: <ul class="list"> + ....: <li>1</li> + ....: <li>2</li> + ....: <li>3</li> + ....: </ul> + ....: <ul class="list"> + ....: <li>4</li> + ....: <li>5</li> + ....: <li>6</li> + ....: </ul>""") + >>> xp = lambda x: sel.xpath(x).getall() + +This gets all first ``<li>`` elements under whatever it is its parent:: + + >>> xp("//li[1]") + ['<li>1</li>', '<li>4</li>'] + +And this gets the first ``<li>`` element in the whole document:: + + >>> xp("(//li)[1]") + ['<li>1</li>'] + +This gets all first ``<li>`` elements under an ``<ul>`` parent:: + + >>> xp("//ul/li[1]") + ['<li>1</li>', '<li>4</li>'] + +And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document:: + + >>> xp("(//ul/li)[1]") + ['<li>1</li>'] + +Using text nodes in a condition +------------------------------- + +When you need to use the text content as argument to an `XPath string function`_, +avoid using ``.//text()`` and use just ``.`` instead. + +This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*. +And when a node-set is converted to a string, which happens when it is passed as argument to +a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only. + +Example:: + + >>> from scrapy import Selector + >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>') + +Converting a *node-set* to string:: + + >>> sel.xpath('//a//text()').getall() # take a peek at the node-set + ['Click here to go to the ', 'Next Page'] + >>> sel.xpath("string(//a[1]//text())").getall() # convert it to string + ['Click here to go to the '] + +A *node* converted to a string, however, puts together the text of itself plus of all its descendants:: + + >>> sel.xpath("//a[1]").getall() # select the first node + ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] + >>> sel.xpath("string(//a[1])").getall() # convert it to string + ['Click here to go to the Next Page'] + +So, using the ``.//text()`` node-set won't select anything in this case:: + + >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").getall() + [] + +But using the ``.`` to mean the node, works:: + + >>> sel.xpath("//a[contains(., 'Next Page')]").getall() + ['<a href="#">Click here to go to the <strong>Next Page</strong></a>'] + +.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions + .. _topics-selectors-xpath-variables: Variables in XPath expressions @@ -298,14 +630,14 @@ Here's an example to match an element based on its "id" attribute value, without hard-coding it (that was shown previously):: >>> # `$val` used in the expression, a `val` argument needs to be passed - >>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first() - u'Name: My image 1 ' + >>> response.xpath('//div[@id=$val]/a/text()', val='images').get() + 'Name: My image 1 ' Here's another example, to find the "id" attribute of a ``<div>`` tag containing five ``<a>`` children (here we pass the value ``5`` as an integer):: - >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first() - u'images' + >>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).get() + 'images' All variable references must have a binding value when calling ``.xpath()`` (otherwise you'll get a ``ValueError: XPath error:`` exception). @@ -314,13 +646,78 @@ This is done by passing as many named arguments as necessary. `parsel`_, the library powering Scrapy selectors, has more details and examples on `XPath variables`_. -.. _parsel: https://parsel.readthedocs.io/ .. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions + +.. _removing-namespaces: + +Removing namespaces +------------------- + +When dealing with scraping projects, it is often quite convenient to get rid of +namespaces altogether and just work with element names, to write more +simple/convenient XPaths. You can use the +:meth:`Selector.remove_namespaces` method for that. + +Let's show an example that illustrates this with the Python Insider blog atom feed. + +.. highlight:: sh + +First, we open the shell with the url we want to scrape:: + + $ scrapy shell https://feeds.feedburner.com/PythonInsider + +This is how the file starts:: + + <?xml version="1.0" encoding="UTF-8"?> + <?xml-stylesheet ... + <feed xmlns="http://www.w3.org/2005/Atom" + xmlns:openSearch="http://a9.com/-/spec/opensearchrss/1.0/" + xmlns:blogger="http://schemas.google.com/blogger/2008" + xmlns:georss="http://www.georss.org/georss" + xmlns:gd="http://schemas.google.com/g/2005" + xmlns:thr="http://purl.org/syndication/thread/1.0" + xmlns:feedburner="http://rssnamespace.org/feedburner/ext/1.0"> + ... + +You can see several namespace declarations including a default +"http://www.w3.org/2005/Atom" and another one using the "gd:" prefix for +"http://schemas.google.com/g/2005". + +.. highlight:: python + +Once in the shell we can try selecting all ``<link>`` objects and see that it +doesn't work (because the Atom XML namespace is obfuscating those nodes):: + + >>> response.xpath("//link") + [] + +But once we call the :meth:`Selector.remove_namespaces` method, all +nodes can be accessed directly by their names:: + + >>> response.selector.remove_namespaces() + >>> response.xpath("//link") + [<Selector xpath='//link' data='<link rel="alternate" type="text/html" h'>, + <Selector xpath='//link' data='<link rel="next" type="application/atom+'>, + ... + +If you wonder why the namespace removal procedure isn't always called by default +instead of having to call it manually, this is because of two reasons, which, in order +of relevance, are: + +1. Removing namespaces requires to iterate and modify all nodes in the + document, which is a reasonably expensive operation to perform by default + for all documents crawled by Scrapy + +2. There could be some cases where using namespaces is actually required, in + case some element names clash between namespaces. These cases are very rare + though. + + Using EXSLT extensions ---------------------- -Being built atop `lxml`_, Scrapy selectors also support some `EXSLT`_ extensions +Being built atop `lxml`_, Scrapy selectors support some `EXSLT`_ extensions and come with these pre-registered namespaces to use in XPath expressions: @@ -340,7 +737,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's Example selecting links in list item with a "class" attribute ending with a digit:: >>> from scrapy import Selector - >>> doc = """ + >>> doc = u""" ... <div> ... <ul> ... <li class="item-0"><a href="link1.html">first item</a></li> @@ -352,10 +749,10 @@ Example selecting links in list item with a "class" attribute ending with a digi ... </div> ... """ >>> sel = Selector(text=doc, type="html") - >>> sel.xpath('//li//@href').extract() - [u'link1.html', u'link2.html', u'link3.html', u'link4.html', u'link5.html'] - >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').extract() - [u'link1.html', u'link2.html', u'link4.html', u'link5.html'] + >>> sel.xpath('//li//@href').getall() + ['link1.html', 'link2.html', 'link3.html', 'link4.html', 'link5.html'] + >>> sel.xpath('//li[re:test(@class, "item-\d$")]//@href').getall() + ['link1.html', 'link2.html', 'link4.html', 'link5.html'] >>> .. warning:: C library ``libxslt`` doesn't natively support EXSLT regular @@ -372,7 +769,7 @@ extracting text elements for example. Example extracting microdata (sample content taken from http://schema.org/Product) with groups of itemscopes and corresponding itemprops:: - >>> doc = """ + >>> doc = u""" ... <div itemscope itemtype="http://schema.org/Product"> ... <span itemprop="name">Kenmore White 17" Microwave</span> ... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' /> @@ -424,33 +821,33 @@ with groups of itemscopes and corresponding itemprops:: ... """ >>> sel = Selector(text=doc, type="html") >>> for scope in sel.xpath('//div[@itemscope]'): - ... print "current scope:", scope.xpath('@itemtype').extract() + ... print("current scope:", scope.xpath('@itemtype').getall()) ... props = scope.xpath(''' ... set:difference(./descendant::*/@itemprop, ... .//*[@itemscope]/*/@itemprop)''') - ... print " properties:", props.extract() - ... print + ... print(" properties: %s" % (props.getall())) + ... print("") - current scope: [u'http://schema.org/Product'] - properties: [u'name', u'aggregateRating', u'offers', u'description', u'review', u'review'] + current scope: ['http://schema.org/Product'] + properties: ['name', 'aggregateRating', 'offers', 'description', 'review', 'review'] - current scope: [u'http://schema.org/AggregateRating'] - properties: [u'ratingValue', u'reviewCount'] + current scope: ['http://schema.org/AggregateRating'] + properties: ['ratingValue', 'reviewCount'] - current scope: [u'http://schema.org/Offer'] - properties: [u'price', u'availability'] + current scope: ['http://schema.org/Offer'] + properties: ['price', 'availability'] - current scope: [u'http://schema.org/Review'] - properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + current scope: ['http://schema.org/Review'] + properties: ['name', 'author', 'datePublished', 'reviewRating', 'description'] - current scope: [u'http://schema.org/Rating'] - properties: [u'worstRating', u'ratingValue', u'bestRating'] + current scope: ['http://schema.org/Rating'] + properties: ['worstRating', 'ratingValue', 'bestRating'] - current scope: [u'http://schema.org/Review'] - properties: [u'name', u'author', u'datePublished', u'reviewRating', u'description'] + current scope: ['http://schema.org/Review'] + properties: ['name', 'author', 'datePublished', 'reviewRating', 'description'] - current scope: [u'http://schema.org/Rating'] - properties: [u'worstRating', u'ratingValue', u'bestRating'] + current scope: ['http://schema.org/Rating'] + properties: ['worstRating', 'ratingValue', 'bestRating'] >>> @@ -462,127 +859,44 @@ inside another ``itemscope``. .. _regular expressions: http://exslt.org/regexp/index.html .. _set manipulation: http://exslt.org/set/index.html +Other XPath extensions +---------------------- -Some XPath tips ---------------- +Scrapy selectors also provide a sorely missed XPath extension function +``has-class`` that returns ``True`` for nodes that have all of the specified +HTML classes. -Here are some tips that you may find useful when using XPath -with Scrapy selectors, based on `this post from ScrapingHub's blog`_. -If you are not much familiar with XPath yet, -you may want to take a look first at this `XPath tutorial`_. +.. highlight:: html +For the following HTML:: -.. _`XPath tutorial`: http://www.zvon.org/comp/r/tut-XPath_1.html -.. _`this post from ScrapingHub's blog`: https://blog.scrapinghub.com/2014/07/17/xpath-tips-from-the-web-scraping-trenches/ + <p class="foo bar-baz">First</p> + <p class="foo">Second</p> + <p class="bar">Third</p> + <p>Fourth</p> +.. highlight:: python -Using text nodes in a condition -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +You can use it like this:: -When you need to use the text content as argument to an `XPath string function`_, -avoid using ``.//text()`` and use just ``.`` instead. - -This is because the expression ``.//text()`` yields a collection of text elements -- a *node-set*. -And when a node-set is converted to a string, which happens when it is passed as argument to -a string function like ``contains()`` or ``starts-with()``, it results in the text for the first element only. - -Example:: - - >>> from scrapy import Selector - >>> sel = Selector(text='<a href="#">Click here to go to the <strong>Next Page</strong></a>') - -Converting a *node-set* to string:: - - >>> sel.xpath('//a//text()').extract() # take a peek at the node-set - [u'Click here to go to the ', u'Next Page'] - >>> sel.xpath("string(//a[1]//text())").extract() # convert it to string - [u'Click here to go to the '] - -A *node* converted to a string, however, puts together the text of itself plus of all its descendants:: - - >>> sel.xpath("//a[1]").extract() # select the first node - [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>'] - >>> sel.xpath("string(//a[1])").extract() # convert it to string - [u'Click here to go to the Next Page'] - -So, using the ``.//text()`` node-set won't select anything in this case:: - - >>> sel.xpath("//a[contains(.//text(), 'Next Page')]").extract() + >>> response.xpath('//p[has-class("foo")]') + [<Selector xpath='//p[has-class("foo")]' data='<p class="foo bar-baz">First</p>'>, + <Selector xpath='//p[has-class("foo")]' data='<p class="foo">Second</p>'>] + >>> response.xpath('//p[has-class("foo", "bar-baz")]') + [<Selector xpath='//p[has-class("foo", "bar-baz")]' data='<p class="foo bar-baz">First</p>'>] + >>> response.xpath('//p[has-class("foo", "bar")]') [] -But using the ``.`` to mean the node, works:: +So XPath ``//p[has-class("foo", "bar-baz")]`` is roughly equivalent to CSS +``p.foo.bar-baz``. Please note, that it is slower in most of the cases, +because it's a pure-Python function that's invoked for every node in question +whereas the CSS lookup is translated into XPath and thus runs more efficiently, +so performance-wise its uses are limited to situations that are not easily +described with CSS selectors. - >>> sel.xpath("//a[contains(., 'Next Page')]").extract() - [u'<a href="#">Click here to go to the <strong>Next Page</strong></a>'] +Parsel also simplifies adding your own XPath extensions. -.. _`XPath string function`: https://www.w3.org/TR/xpath/#section-String-Functions - -Beware of the difference between //node[1] and (//node)[1] -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``//node[1]`` selects all the nodes occurring first under their respective parents. - -``(//node)[1]`` selects all the nodes in the document, and then gets only the first of them. - -Example:: - - >>> from scrapy import Selector - >>> sel = Selector(text=""" - ....: <ul class="list"> - ....: <li>1</li> - ....: <li>2</li> - ....: <li>3</li> - ....: </ul> - ....: <ul class="list"> - ....: <li>4</li> - ....: <li>5</li> - ....: <li>6</li> - ....: </ul>""") - >>> xp = lambda x: sel.xpath(x).extract() - -This gets all first ``<li>`` elements under whatever it is its parent:: - - >>> xp("//li[1]") - [u'<li>1</li>', u'<li>4</li>'] - -And this gets the first ``<li>`` element in the whole document:: - - >>> xp("(//li)[1]") - [u'<li>1</li>'] - -This gets all first ``<li>`` elements under an ``<ul>`` parent:: - - >>> xp("//ul/li[1]") - [u'<li>1</li>', u'<li>4</li>'] - -And this gets the first ``<li>`` element under an ``<ul>`` parent in the whole document:: - - >>> xp("(//ul/li)[1]") - [u'<li>1</li>'] - -When querying by class, consider using CSS -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Because an element can contain multiple CSS classes, the XPath way to select elements -by class is the rather verbose:: - - *[contains(concat(' ', normalize-space(@class), ' '), ' someclass ')] - -If you use ``@class='someclass'`` you may end up missing elements that have -other classes, and if you just use ``contains(@class, 'someclass')`` to make up -for that you may end up with more elements that you want, if they have a different -class name that shares the string ``someclass``. - -As it turns out, Scrapy selectors allow you to chain selectors, so most of the time -you can just select by class using CSS and then switch to XPath when needed:: - - >>> from scrapy import Selector - >>> sel = Selector(text='<div class="hero shout"><time datetime="2014-07-23 19:00">Special date</time></div>') - >>> sel.css('.shout').xpath('./time/@datetime').extract() - [u'2014-07-23 19:00'] - -This is cleaner than using the verbose XPath trick shown above. Just remember -to use the ``.`` in the XPath expressions that will follow. +.. autofunction:: parsel.xpathfuncs.set_xpathfunc .. _topics-selectors-ref: @@ -596,132 +910,79 @@ Built-in Selectors reference Selector objects ---------------- -.. class:: Selector(response=None, text=None, type=None) +.. autoclass:: Selector - An instance of :class:`Selector` is a wrapper over response to select - certain parts of its content. - - ``response`` is an :class:`~scrapy.http.HtmlResponse` or an - :class:`~scrapy.http.XmlResponse` object that will be used for selecting and - extracting data. - - ``text`` is a unicode string or utf-8 encoded text for cases when a - ``response`` isn't available. Using ``text`` and ``response`` together is - undefined behavior. - - ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default). - - If ``type`` is ``None``, the selector automatically chooses the best type - based on ``response`` type (see below), or defaults to ``"html"`` in case it - is used together with ``text``. - - If ``type`` is ``None`` and a ``response`` is passed, the selector type is - inferred from the response type as follows: - - * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type - * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type - * ``"html"`` for anything else - - Otherwise, if ``type`` is set, the selector type will be forced and no - detection will occur. - - .. method:: xpath(query) - - Find nodes matching the xpath ``query`` and return the result as a - :class:`SelectorList` instance with all elements flattened. List - elements implement :class:`Selector` interface too. - - ``query`` is a string containing the XPATH query to apply. + .. automethod:: xpath .. note:: For convenience, this method can be called as ``response.xpath()`` - .. method:: css(query) - - Apply the given CSS selector and return a :class:`SelectorList` instance. - - ``query`` is a string containing the CSS selector to apply. - - In the background, CSS queries are translated into XPath queries using - `cssselect`_ library and run ``.xpath()`` method. + .. automethod:: css .. note:: - For convenience this method can be called as ``response.css()`` + For convenience, this method can be called as ``response.css()`` - .. method:: extract() + .. automethod:: get - Serialize and return the matched nodes as a list of unicode strings. - Percent encoded content is unquoted. + See also: :ref:`old-extraction-api` - .. method:: re(regex) + .. autoattribute:: attrib - Apply the given regex and return a list of unicode strings with the - matches. + See also: :ref:`selecting-attributes`. - ``regex`` can be either a compiled regular expression or a string which - will be compiled to a regular expression using ``re.compile(regex)`` + .. automethod:: re - .. note:: + .. automethod:: re_first - Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``<`` and ``&``). + .. automethod:: register_namespace - .. method:: register_namespace(prefix, uri) + .. automethod:: remove_namespaces - Register the given namespace to be used in this :class:`Selector`. - Without registering namespaces you can't select or extract data from - non-standard namespaces. See examples below. + .. automethod:: __bool__ - .. method:: remove_namespaces() - - Remove all namespaces, allowing to traverse the document using - namespace-less xpaths. See example below. - - .. method:: __nonzero__() - - Returns ``True`` if there is any real content selected or ``False`` - otherwise. In other words, the boolean value of a :class:`Selector` is - given by the contents it selects. + .. automethod:: getall + This method is added to Selector for consistency; it is more useful + with SelectorList. See also: :ref:`old-extraction-api` SelectorList objects -------------------- -.. class:: SelectorList +.. autoclass:: SelectorList - The :class:`SelectorList` class is a subclass of the builtin ``list`` - class, which provides a few additional methods. + .. automethod:: xpath - .. method:: xpath(query) + .. automethod:: css - Call the ``.xpath()`` method for each element in this list and return - their results flattened as another :class:`SelectorList`. + .. automethod:: getall - ``query`` is the same argument as the one in :meth:`Selector.xpath` + See also: :ref:`old-extraction-api` - .. method:: css(query) + .. automethod:: get - Call the ``.css()`` method for each element in this list and return - their results flattened as another :class:`SelectorList`. + See also: :ref:`old-extraction-api` - ``query`` is the same argument as the one in :meth:`Selector.css` + .. automethod:: re - .. method:: extract() + .. automethod:: re_first - Call the ``.extract()`` method for each element in this list and return - their results flattened, as a list of unicode strings. + .. autoattribute:: attrib - .. method:: re() + See also: :ref:`selecting-attributes`. - Call the ``.re()`` method for each element in this list and return - their results flattened, as a list of unicode strings. +.. _selector-examples: +Examples +======== + +.. _selector-examples-html: Selector examples on HTML response ---------------------------------- -Here's a couple of :class:`Selector` examples to illustrate several concepts. +Here are some :class:`Selector` examples to illustrate several concepts. In all cases, we assume there is already a :class:`Selector` instantiated with a :class:`~scrapy.http.HtmlResponse` object like this:: @@ -735,20 +996,22 @@ a :class:`~scrapy.http.HtmlResponse` object like this:: 2. Extract the text of all ``<h1>`` elements from an HTML response body, returning a list of unicode strings:: - sel.xpath("//h1").extract() # this includes the h1 tag - sel.xpath("//h1/text()").extract() # this excludes the h1 tag + sel.xpath("//h1").getall() # this includes the h1 tag + sel.xpath("//h1/text()").getall() # this excludes the h1 tag 3. Iterate over all ``<p>`` tags and print their class attribute:: for node in sel.xpath("//p"): - print node.xpath("@class").extract() + print(node.attrib['class']) + + +.. _selector-examples-xml: Selector examples on XML response --------------------------------- -Here's a couple of examples to illustrate several concepts. In both cases we -assume there is already a :class:`Selector` instantiated with an -:class:`~scrapy.http.XmlResponse` object like this:: +Here are some examples to illustrate concepts for :class:`Selector` objects +instantiated with an :class:`~scrapy.http.XmlResponse` object:: sel = Selector(xml_response) @@ -761,53 +1024,6 @@ assume there is already a :class:`Selector` instantiated with an a namespace:: sel.register_namespace("g", "http://base.google.com/ns/1.0") - sel.xpath("//g:price").extract() - -.. _removing-namespaces: - -Removing namespaces -------------------- - -When dealing with scraping projects, it is often quite convenient to get rid of -namespaces altogether and just work with element names, to write more -simple/convenient XPaths. You can use the -:meth:`Selector.remove_namespaces` method for that. - -Let's show an example that illustrates this with GitHub blog atom feed. - -.. highlight:: sh - -First, we open the shell with the url we want to scrape:: - - $ scrapy shell https://github.com/blog.atom - -.. highlight:: python - -Once in the shell we can try selecting all ``<link>`` objects and see that it -doesn't work (because the Atom XML namespace is obfuscating those nodes):: - - >>> response.xpath("//link") - [] - -But once we call the :meth:`Selector.remove_namespaces` method, all -nodes can be accessed directly by their names:: - - >>> response.selector.remove_namespaces() - >>> response.xpath("//link") - [<Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>, - <Selector xpath='//link' data=u'<link xmlns="http://www.w3.org/2005/Atom'>, - ... - -If you wonder why the namespace removal procedure isn't always called by default -instead of having to call it manually, this is because of two reasons, which, in order -of relevance, are: - -1. Removing namespaces requires to iterate and modify all nodes in the - document, which is a reasonably expensive operation to perform for all - documents crawled by Scrapy - -2. There could be some cases where using namespaces is actually required, in - case some element names clash between namespaces. These cases are very rare - though. + sel.xpath("//g:price").getall() .. _Google Base XML feed: https://support.google.com/merchants/answer/160589?hl=en&ref_topic=2473799 diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1f1217770..47b6cf13d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -871,7 +871,7 @@ LOG_STDOUT Default: ``False`` If ``True``, all standard output (and error) of your process will be redirected -to the log. For example if you ``print 'hello'`` it will appear in the Scrapy +to the log. For example if you ``print('hello')`` it will appear in the Scrapy log. .. setting:: LOG_SHORT_NAMES diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 11ab199f2..68a0b19b5 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -179,13 +179,13 @@ all start with the ``[s]`` prefix):: After that, we can start playing with the objects:: - >>> response.xpath('//title/text()').extract_first() + >>> response.xpath('//title/text()').get() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("https://reddit.com") - >>> response.xpath('//title/text()').extract() - ['reddit: the front page of the internet'] + >>> response.xpath('//title/text()').get() + 'reddit: the front page of the internet' >>> request = request.replace(method="POST") diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d40c0e1df..ff07b9d55 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -279,6 +279,22 @@ request_dropped :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_reached_downloader +--------------------------- + +.. signal:: request_reached_downloader +.. function:: request_reached_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request` reached downloader. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 697732b47..a08dc30f2 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback:: ] def parse(self, response): - for h3 in response.xpath('//h3').extract(): + for h3 in response.xpath('//h3').getall(): yield {"title": h3} - for url in response.xpath('//a/@href').extract(): - yield scrapy.Request(url, callback=self.parse) + for href in response.xpath('//a/@href').getall(): + yield scrapy.Request(response.urljoin(href), self.parse) Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly; to give data more structure you can use :ref:`topics-items`:: @@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`:: yield scrapy.Request('http://www.example.com/3.html', self.parse) def parse(self, response): - for h3 in response.xpath('//h3').extract(): + for h3 in response.xpath('//h3').getall(): yield MyItem(title=h3) - for url in response.xpath('//a/@href').extract(): - yield scrapy.Request(url, callback=self.parse) + for href in response.xpath('//a/@href').getall(): + yield scrapy.Request(response.urljoin(href), self.parse) .. _spiderargs: @@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules:: self.logger.info('Hi, this is an item page! %s', response.url) item = scrapy.Item() item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)') - item['name'] = response.xpath('//td[@id="item_name"]/text()').extract() - item['description'] = response.xpath('//td[@id="item_description"]/text()').extract() + item['name'] = response.xpath('//td[@id="item_name"]/text()').get() + item['description'] = response.xpath('//td[@id="item_description"]/text()').get() return item @@ -545,12 +545,12 @@ These spiders are pretty easy to use, let's have a look at one example:: itertag = 'item' def parse_node(self, response, node): - self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract())) + self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall())) item = TestItem() - item['id'] = node.xpath('@id').extract() - item['name'] = node.xpath('name').extract() - item['description'] = node.xpath('description').extract() + item['id'] = node.xpath('@id').get() + item['name'] = node.xpath('name').get() + item['description'] = node.xpath('description').get() return item Basically what we did up there was to create a spider that downloads a feed from diff --git a/requirements-py2.txt b/requirements-py2.txt index 03b33d02d..0771aae3a 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -6,5 +6,5 @@ queuelib w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -parsel>=1.4 +parsel>=1.5 service_identity diff --git a/requirements-py3.txt b/requirements-py3.txt index b38c4cc09..5a5d4c95a 100644 --- a/requirements-py3.txt +++ b/requirements-py3.txt @@ -6,5 +6,5 @@ queuelib>=1.1.1 w3lib>=1.17.0 six>=1.5.2 PyDispatcher>=2.0.5 -parsel>=1.4 +parsel>=1.5 service_identity diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index de7ac4503..259220a72 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -42,23 +42,38 @@ class ContractsManager(object): requests = [] for method in self.tested_methods_from_spidercls(type(spider)): bound_method = spider.__getattribute__(method) - requests.append(self.from_method(bound_method, results)) + try: + requests.append(self.from_method(bound_method, results)) + except Exception: + case = _create_testcase(bound_method, 'contract') + results.addError(case, sys.exc_info()) return requests def from_method(self, method, results): contracts = self.extract_contracts(method) if contracts: + request_cls = Request + for contract in contracts: + if contract.request_cls is not None: + request_cls = contract.request_cls + # calculate request args - args, kwargs = get_spec(Request.__init__) + args, kwargs = get_spec(request_cls.__init__) + + # Don't filter requests to allow + # testing different callbacks on the same URL. + kwargs['dont_filter'] = True kwargs['callback'] = method + for contract in contracts: kwargs = contract.adjust_request_args(kwargs) - # create and prepare request args.remove('self') + + # check if all positional arguments are defined in kwargs if set(args).issubset(set(kwargs)): - request = Request(**kwargs) + request = request_cls(**kwargs) # execute pre and post hooks in order for contract in reversed(contracts): @@ -94,6 +109,7 @@ class ContractsManager(object): class Contract(object): """ Abstract class for contracts """ + request_cls = None def __init__(self, method, *args): self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d835e65f7..59c3ad074 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -129,6 +129,9 @@ class Downloader(object): return response slot.active.add(request) + self.signals.send_catch_log(signal=signals.request_reached_downloader, + request=request, + spider=spider) deferred = defer.Deferred().addBoth(_deactivate) slot.queue.append((request, deferred)) self._process_queue(spider, slot) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index f4ca4262a..5fa6b771c 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor): base_url = get_base_url(response) body = u''.join(f for x in self.restrict_xpaths - for f in response.xpath(x).extract() + for f in response.xpath(x).getall() ).encode(response.encoding, errors='xmlcharrefreplace') else: body = response.body diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index e73413318..a7c75a46a 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -181,7 +181,7 @@ class ItemLoader(object): def _get_xpathvalues(self, xpaths, **kw): self._check_selector_method() xpaths = arg_to_iter(xpaths) - return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths) + return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths) def add_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) @@ -198,6 +198,6 @@ class ItemLoader(object): def _get_cssvalues(self, csss, **kw): self._check_selector_method() csss = arg_to_iter(csss) - return flatten(self.selector.css(css).extract() for css in csss) + return flatten(self.selector.css(css).getall() for css in csss) XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 64cb0232c..8f6cb1d79 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -27,6 +27,10 @@ def _response_from_text(text, st): class SelectorList(_ParselSelector.selectorlist_cls, object_ref): + """ + The :class:`SelectorList` class is a subclass of the builtin ``list`` + class, which provides a few additional methods. + """ @deprecated(use_instead='.extract()') def extract_unquoted(self): return [x.extract_unquoted() for x in self] @@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref): class Selector(_ParselSelector, object_ref): + """ + An instance of :class:`Selector` is a wrapper over response to select + certain parts of its content. + + ``response`` is an :class:`~scrapy.http.HtmlResponse` or an + :class:`~scrapy.http.XmlResponse` object that will be used for selecting + and extracting data. + + ``text`` is a unicode string or utf-8 encoded text for cases when a + ``response`` isn't available. Using ``text`` and ``response`` together is + undefined behavior. + + ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` + or ``None`` (default). + + If ``type`` is ``None``, the selector automatically chooses the best type + based on ``response`` type (see below), or defaults to ``"html"`` in case it + is used together with ``text``. + + If ``type`` is ``None`` and a ``response`` is passed, the selector type is + inferred from the response type as follows: + + * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type + * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type + * ``"html"`` for anything else + + Otherwise, if ``type`` is set, the selector type will be forced and no + detection will occur. + """ __slots__ = ['response'] selectorlist_cls = SelectorList diff --git a/scrapy/signals.py b/scrapy/signals.py index e36c27203..c0e4bb74e 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -13,6 +13,7 @@ spider_closed = object() spider_error = object() request_scheduled = object() request_dropped = object() +request_reached_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() diff --git a/scrapy/templates/spiders/crawl.tmpl b/scrapy/templates/spiders/crawl.tmpl index 802cb88a1..878425125 100644 --- a/scrapy/templates/spiders/crawl.tmpl +++ b/scrapy/templates/spiders/crawl.tmpl @@ -14,8 +14,8 @@ class $classname(CrawlSpider): ) def parse_item(self, response): - i = {} - #i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract() - #i['name'] = response.xpath('//div[@id="name"]').extract() - #i['description'] = response.xpath('//div[@id="description"]').extract() - return i + item = {} + #item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get() + #item['name'] = response.xpath('//div[@id="name"]').get() + #item['description'] = response.xpath('//div[@id="description"]').get() + return item diff --git a/scrapy/templates/spiders/xmlfeed.tmpl b/scrapy/templates/spiders/xmlfeed.tmpl index 7c2ff8850..863c9772f 100644 --- a/scrapy/templates/spiders/xmlfeed.tmpl +++ b/scrapy/templates/spiders/xmlfeed.tmpl @@ -10,8 +10,8 @@ class $classname(XMLFeedSpider): itertag = 'item' # change it accordingly def parse_node(self, response, selector): - i = {} - #i['url'] = selector.select('url').extract() - #i['name'] = selector.select('name').extract() - #i['description'] = selector.select('description').extract() - return i + item = {} + #item['url'] = selector.select('url').get() + #item['name'] = selector.select('name').get() + #item['description'] = selector.select('description').get() + return item diff --git a/setup.py b/setup.py index c37919cda..8c47f67ce 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=1.4', + 'parsel>=1.5', 'PyDispatcher>=2.0.5', 'service_identity', ], diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 3e27d6abd..36baacfbd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_response_selector_html(self): - xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]' + xpath = 'response.xpath("//p[@class=\'one\']/text()").get()' _, out, _ = yield self.execute([self.url('/html'), '-c', xpath]) self.assertEqual(out.strip(), b'Works') diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 322d20c4c..a06bb2cc3 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,18 +1,23 @@ from unittest import TextTestResult +from six import get_unbound_function +from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest +from scrapy import FormRequest +from scrapy.crawler import CrawlerRunner from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.http import Request from scrapy.item import Item, Field -from scrapy.contracts import ContractsManager +from scrapy.contracts import ContractsManager, Contract from scrapy.contracts.default import ( UrlContract, ReturnsContract, ScrapesContract, ) +from tests.mockserver import MockServer class TestItem(Item): @@ -24,6 +29,30 @@ class ResponseMock(object): url = 'http://scrapy.org' +class CustomSuccessContract(Contract): + name = 'custom_success_contract' + + def adjust_request_args(self, args): + args['url'] = 'http://scrapy.org' + return args + + +class CustomFailContract(Contract): + name = 'custom_fail_contract' + + def adjust_request_args(self, args): + raise TypeError('Error in adjust_request_args') + + +class CustomFormContract(Contract): + name = 'custom_form' + request_cls = FormRequest + + def adjust_request_args(self, args): + args['formdata'] = {'name': 'scrapy'} + return args + + class TestSpider(Spider): name = 'demo_spider' @@ -100,13 +129,47 @@ class TestSpider(Spider): """ pass + def custom_form(self, response): + """ + @url http://scrapy.org + @custom_form + """ + pass + + +class CustomContractSuccessSpider(Spider): + name = 'custom_contract_success_spider' + + def parse(self, response): + """ + @custom_success_contract + """ + pass + + +class CustomContractFailSpider(Spider): + name = 'custom_contract_fail_spider' + + def parse(self, response): + """ + @custom_fail_contract + """ + pass + class InheritsTestSpider(TestSpider): name = 'inherits_demo_spider' class ContractsManagerTest(unittest.TestCase): - contracts = [UrlContract, ReturnsContract, ScrapesContract] + contracts = [ + UrlContract, + ReturnsContract, + ScrapesContract, + CustomFormContract, + CustomSuccessContract, + CustomFailContract, + ] def setUp(self): self.conman = ContractsManager(self.contracts) @@ -120,6 +183,9 @@ class ContractsManagerTest(unittest.TestCase): self.assertTrue(self.results.failures) self.assertFalse(self.results.errors) + def should_error(self): + self.assertTrue(self.results.errors) + def test_contracts(self): spider = TestSpider() @@ -181,17 +247,22 @@ class ContractsManagerTest(unittest.TestCase): self.should_succeed() # scrapes_item_fail - request = self.conman.from_method(spider.scrapes_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_item_fail, self.results) request.callback(response) self.should_fail() # scrapes_dict_item_fail - request = self.conman.from_method(spider.scrapes_dict_item_fail, - self.results) + request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results) request.callback(response) self.should_fail() + def test_custom_contracts(self): + self.conman.from_spider(CustomContractSuccessSpider(), self.results) + self.should_succeed() + + self.conman.from_spider(CustomContractFailSpider(), self.results) + self.should_error() + def test_errback(self): spider = TestSpider() response = ResponseMock() @@ -207,6 +278,44 @@ class ContractsManagerTest(unittest.TestCase): self.assertFalse(self.results.failures) self.assertTrue(self.results.errors) + @defer.inlineCallbacks + def test_same_url(self): + + class TestSameUrlSpider(Spider): + name = 'test_same_url' + + def __init__(self, *args, **kwargs): + super(TestSameUrlSpider, self).__init__(*args, **kwargs) + self.visited = 0 + + def start_requests(s): + return self.conman.from_spider(s, self.results) + + def parse_first(self, response): + self.visited += 1 + return TestItem() + + def parse_second(self, response): + self.visited += 1 + return TestItem() + + with MockServer() as mockserver: + contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) + + get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc + get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc + + crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) + yield crawler.crawl() + + self.assertEqual(crawler.spider.visited, 2) + + def test_form_contract(self): + spider = TestSpider() + request = self.conman.from_method(spider.custom_form, self.results) + self.assertEqual(request.method, 'POST') + self.assertIsInstance(request, FormRequest) + def test_inherited_contracts(self): spider = InheritsTestSpider() diff --git a/tests/test_engine.py b/tests/test_engine.py index 719c0c60c..856465161 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -103,6 +103,7 @@ class CrawlerRun(object): self.respplug = [] self.reqplug = [] self.reqdropped = [] + self.reqreached = [] self.itemerror = [] self.itemresp = [] self.signals_catched = {} @@ -124,6 +125,7 @@ class CrawlerRun(object): self.crawler.signals.connect(self.item_error, signals.item_error) self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled) self.crawler.signals.connect(self.request_dropped, signals.request_dropped) + self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader) self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded) self.crawler.crawl(start_urls=start_urls) self.spider = self.crawler.spider @@ -155,6 +157,9 @@ class CrawlerRun(object): def request_scheduled(self, request, spider): self.reqplug.append((request, spider)) + def request_reached(self, request, spider): + self.reqreached.append((request, spider)) + def request_dropped(self, request, spider): self.reqdropped.append((request, spider)) @@ -212,6 +217,8 @@ class EngineTest(unittest.TestCase): responses_count = len(self.run.respplug) self.assertEqual(scheduled_requests_count, dropped_requests_count + responses_count) + self.assertEqual(len(self.run.reqreached), + responses_count) def _assert_dropped_requests(self): self.assertEqual(len(self.run.reqdropped), 1) @@ -219,6 +226,7 @@ class EngineTest(unittest.TestCase): def _assert_downloaded_responses(self): # response tests self.assertEqual(8, len(self.run.respplug)) + self.assertEqual(8, len(self.run.reqreached)) for response, _ in self.run.respplug: if self.run.getpath(response.url) == '/item999.html': diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 820758dc9..3b90e3dac 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//title/text()").extract(), + response.selector.xpath("//title/text()").getall(), [u'Some page'] ) self.assertEqual( - response.selector.css("title::text").extract(), + response.selector.css("title::text").getall(), [u'Some page'] ) self.assertEqual( @@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//title/text()").extract(), - response.selector.xpath("//title/text()").extract(), + response.xpath("//title/text()").getall(), + response.selector.xpath("//title/text()").getall(), ) self.assertEqual( - response.css("title::text").extract(), - response.selector.css("title::text").extract(), + response.css("title::text").getall(), + response.selector.css("title::text").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(), - response.xpath("normalize-space(//p[@class=\"content\"])").extract(), + response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(), + response.xpath("normalize-space(//p[@class=\"content\"])").getall(), ) self.assertEqual( response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()", - pclass="content", pcount=1).extract(), - response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(), + pclass="content", pcount=1).getall(), + response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(), ) def test_urljoin_with_base_url(self): @@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest): self.assertIs(response.selector.response, response) self.assertEqual( - response.selector.xpath("//elem/text()").extract(), + response.selector.xpath("//elem/text()").getall(), [u'value'] ) @@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//elem/text()").extract(), - response.selector.xpath("//elem/text()").extract(), + response.xpath("//elem/text()").getall(), + response.selector.xpath("//elem/text()").getall(), ) def test_selector_shortcuts_kwargs(self): @@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest): response = self.response_class("http://www.example.com", body=body) self.assertEqual( - response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(), + response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(), ) response.selector.register_namespace('s2', 'http://scrapy.org') self.assertEqual( - response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(), - response.selector.xpath("//s2:elem/text()").extract(), + response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(), + response.selector.xpath("//s2:elem/text()").getall(), ) diff --git a/tests/test_loader.py b/tests/test_loader.py index 3b5714058..8b58e4dbd 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_xpath("//header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>']) @@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase): nl = l.nested_css("header") nl.add_xpath('name', 'div/text()') nl.add_css('name_div', '#id') - nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract()) + nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall()) self.assertEqual(l.get_output_value('name'), [u'marta']) self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>']) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 5985a6f3e..fb72c9d6d 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider): for href in response.xpath(''' //table[thead/tr/th="Filename"] /tbody//a/@href - ''').extract()], + ''').getall()], } yield item diff --git a/tests/test_selector.py b/tests/test_selector.py index 526660cc8..bc4baf7ea 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase): for x in xl: assert isinstance(x, Selector) - self.assertEqual(sel.xpath('//input').extract(), - [x.extract() for x in sel.xpath('//input')]) + self.assertEqual(sel.xpath('//input').getall(), + [x.get() for x in sel.xpath('//input')]) - self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")], + self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")], [u'a']) - self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], + self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")], [u'12.0']) - self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(), + self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(), [u'xpathrules']) - self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], + self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")], [u'12']) def test_root_base_url(self): @@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase): text = b'<div><img src="a.jpg"><p>Hello</div>' sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'xml') - self.assertEqual(sel.xpath("//div").extract(), + self.assertEqual(sel.xpath("//div").getall(), [u'<div><img src="a.jpg"><p>Hello</p></img></div>']) sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8')) self.assertEqual(sel.type, 'html') - self.assertEqual(sel.xpath("//div").extract(), + self.assertEqual(sel.xpath("//div").getall(), [u'<div><img src="a.jpg"><p>Hello</p></div>']) def test_http_header_encoding_precedence(self): @@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase): headers = {'Content-Type': ['text/html; charset=utf-8']} response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8) x = Selector(response) - self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(), + self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(), [u'\xa3']) def test_badly_encoded_body(self): # \xe9 alone isn't valid utf8 sequence - r1 = TextResponse('http://www.example.com', \ - body=b'<html><p>an Jos\xe9 de</p><html>', \ + r1 = TextResponse('http://www.example.com', + body=b'<html><p>an Jos\xe9 de</p><html>', encoding='utf-8') - Selector(r1).xpath('//text()').extract() + Selector(r1).xpath('//text()').getall() def test_weakref_slots(self): """Check that classes are using slots and are weak-referenceable""" diff --git a/tests/test_spider.py b/tests/test_spider.py index 929e0fea8..f26da2334 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest): def parse_node(self, response, selector): yield { - 'loc': selector.xpath('a:loc/text()').extract(), - 'updated': selector.xpath('b:updated/text()').extract(), - 'other': selector.xpath('other/@value').extract(), - 'custom': selector.xpath('other/@b:custom').extract(), + 'loc': selector.xpath('a:loc/text()').getall(), + 'updated': selector.xpath('b:updated/text()').getall(), + 'other': selector.xpath('other/@value').getall(), + 'custom': selector.xpath('other/@b:custom').getall(), } for iterator in ('iternodes', 'xml'): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index f953076b8..2d845697e 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase): response = XmlResponse(url="http://example.com", body=body) attrs = [] for x in self.xmliter(response, 'product'): - attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract())) + attrs.append(( + x.attrib['id'], + x.xpath("name/text()").getall(), + x.xpath("./type/text()").getall())) self.assertEqual(attrs, - [(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])]) + [('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])]) def test_xmliter_unusual_node(self): body = b"""<?xml version="1.0" encoding="UTF-8"?> @@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase): </root> """ response = XmlResponse(url="http://example.com", body=body) - nodenames = [e.xpath('name()').extract() + nodenames = [e.xpath('name()').getall() for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) @@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase): attrs = [] for x in self.xmliter(r, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + attrs.append((x.attrib['id'], + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(), + x.xpath(u'./tímabil/fyrstaþing/text()').getall())) self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + [(u'26', [u'-'], [u'80']), + (u'21', [u'Ab'], [u'76']), + (u'27', [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" - self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')], + self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')], [[u'one'], [u'two']]) def test_xmliter_namespaces(self): @@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase): node = next(my_iter) node.register_namespace('g', 'http://base.google.com/ns/1.0') - self.assertEqual(node.xpath('title/text()').extract(), ['Item 1']) - self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1']) - self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1']) - self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) - self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1']) - self.assertEqual(node.xpath('g:price/text()').extract(), ['400']) - self.assertEqual(node.xpath('image_link/text()').extract(), []) - self.assertEqual(node.xpath('id/text()').extract(), []) - self.assertEqual(node.xpath('price/text()').extract(), []) + self.assertEqual(node.xpath('title/text()').getall(), ['Item 1']) + self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1']) + self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1']) + self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1']) + self.assertEqual(node.xpath('g:price/text()').getall(), ['400']) + self.assertEqual(node.xpath('image_link/text()').getall(), []) + self.assertEqual(node.xpath('id/text()').getall(), []) + self.assertEqual(node.xpath('price/text()').getall(), []) def test_xmliter_exception(self): body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>""" @@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase): body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n' response = XmlResponse('http://www.example.com', body=body) self.assertEqual( - next(self.xmliter(response, 'item')).extract(), + next(self.xmliter(response, 'item')).get(), u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>' ) @@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase): namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0') node = next(namespace_iter) - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg']) + self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg']) node = next(namespace_iter) - self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg']) + self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg']) def test_xmliter_namespaces_prefix(self): body = b"""\ @@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase): my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h') node = next(my_iter) - self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2) - self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples']) - self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas']) + self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2) + self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples']) + self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas']) my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f') node = next(my_iter) - self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table']) + self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table']) def test_xmliter_objtype_exception(self): i = self.xmliter(42, 'product') diff --git a/tox.ini b/tox.ini index e5543fe2a..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -51,6 +51,9 @@ deps = cssselect==0.9.1 zope.interface==4.1.1 -rtests/requirements-py2.txt +# Not used directly but allows boto GCE plugins to load. +# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + google-compute-engine==2.8.12 [testenv:trunk] basepython = python2.7