mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into telnet-auth
This commit is contained in:
Mikhail Korobov
commit
dbfabf02e8
|
|
@ -12,8 +12,10 @@ dist
|
|||
.idea
|
||||
htmlcov/
|
||||
.coverage
|
||||
.pytest_cache/
|
||||
.coverage.*
|
||||
.cache/
|
||||
.pytest_cache/
|
||||
|
||||
# Windows
|
||||
Thumbs.db
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
language: python
|
||||
sudo: false
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
|
|
|
|||
11
MANIFEST.in
11
MANIFEST.in
|
|
@ -3,13 +3,24 @@ include AUTHORS
|
|||
include INSTALL
|
||||
include LICENSE
|
||||
include MANIFEST.in
|
||||
include NEWS
|
||||
|
||||
include scrapy/VERSION
|
||||
include scrapy/mime.types
|
||||
|
||||
include codecov.yml
|
||||
include conftest.py
|
||||
include pytest.ini
|
||||
include requirements-*.txt
|
||||
include tox.ini
|
||||
|
||||
recursive-include scrapy/templates *
|
||||
recursive-include scrapy license.txt
|
||||
recursive-include docs *
|
||||
prune docs/build
|
||||
|
||||
recursive-include extras *
|
||||
recursive-include bin *
|
||||
recursive-include tests *
|
||||
|
||||
global-exclude __pycache__ *.py[cod]
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the
|
|||
We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv <intro-using-virtualenv>`,
|
||||
to avoid conflicting with your system packages.
|
||||
|
||||
For more detailed and platform specifics instructions, read on.
|
||||
For more detailed and platform specifics instructions, as well as
|
||||
troubleshooting information, read on.
|
||||
|
||||
|
||||
Things that are good to know
|
||||
|
|
@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency.
|
|||
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||
|
||||
|
||||
.. _intro-install-troubleshooting:
|
||||
|
||||
Troubleshooting
|
||||
===============
|
||||
|
||||
AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
|
||||
----------------------------------------------------------------
|
||||
|
||||
After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an
|
||||
exception with the following traceback::
|
||||
|
||||
[…]
|
||||
File "[…]/site-packages/twisted/protocols/tls.py", line 63, in <module>
|
||||
from twisted.internet._sslverify import _setAcceptableProtocols
|
||||
File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in <module>
|
||||
TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1,
|
||||
AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1'
|
||||
|
||||
The reason you get this exception is that your system or virtual environment
|
||||
has a version of pyOpenSSL that your version of Twisted does not support.
|
||||
|
||||
To install a version of pyOpenSSL that your version of Twisted supports,
|
||||
reinstall Twisted with the :code:`tls` extra option::
|
||||
|
||||
pip install twisted[tls]
|
||||
|
||||
For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
|
||||
|
||||
.. _Python: https://www.python.org/
|
||||
.. _pip: https://pip.pypa.io/en/latest/installing/
|
||||
.. _lxml: http://lxml.de/
|
||||
|
|
|
|||
|
|
@ -34,11 +34,11 @@ http://quotes.toscrape.com, following the pagination::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.xpath('span/small/text()').extract_first(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.xpath('span/small/text()').get(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr("href")').extract_first()
|
||||
next_page = response.css('li.next a::attr("href")').get()
|
||||
if next_page is not None:
|
||||
yield response.follow(next_page, self.parse)
|
||||
|
||||
|
|
|
|||
|
|
@ -22,9 +22,7 @@ Scrapy is written in Python_. If you're new to the language you might want to
|
|||
start by getting an idea of what the language is like, to get the most out of
|
||||
Scrapy.
|
||||
|
||||
If you're already familiar with other languages, and want to learn Python
|
||||
quickly, we recommend reading through `Dive Into Python 3`_. Alternatively,
|
||||
you can follow the `Python Tutorial`_.
|
||||
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
|
||||
|
||||
If you're new to programming and want to start with Python, the following books
|
||||
may be useful to you:
|
||||
|
|
@ -40,7 +38,6 @@ as well as the `suggested resources in the learnpython-subreddit`_.
|
|||
|
||||
.. _Python: https://www.python.org/
|
||||
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
|
||||
.. _Dive Into Python 3: http://www.diveintopython3.net
|
||||
.. _Python Tutorial: https://docs.python.org/3/tutorial
|
||||
.. _Automate the Boring Stuff With Python: https://automatetheboringstuff.com/
|
||||
.. _How To Think Like a Computer Scientist: http://openbookproject.net/thinkcs/python/english3e/
|
||||
|
|
@ -254,7 +251,7 @@ data.
|
|||
|
||||
To extract the text from the title above, you can do::
|
||||
|
||||
>>> response.css('title::text').extract()
|
||||
>>> response.css('title::text').getall()
|
||||
['Quotes to Scrape']
|
||||
|
||||
There are two things to note here: one is that we've added ``::text`` to the
|
||||
|
|
@ -262,32 +259,33 @@ CSS query, to mean we want to select only the text elements directly inside
|
|||
``<title>`` element. If we don't specify ``::text``, we'd get the full title
|
||||
element, including its tags::
|
||||
|
||||
>>> response.css('title').extract()
|
||||
>>> response.css('title').getall()
|
||||
['<title>Quotes to Scrape</title>']
|
||||
|
||||
The other thing is that the result of calling ``.extract()`` is a list, because
|
||||
we're dealing with an instance of :class:`~scrapy.selector.SelectorList`. When
|
||||
you know you just want the first result, as in this case, you can do::
|
||||
The other thing is that the result of calling ``.getall()`` is a list: it is
|
||||
possible that a selector returns more than one result, so we extract them all.
|
||||
When you know you just want the first result, as in this case, you can do::
|
||||
|
||||
>>> response.css('title::text').extract_first()
|
||||
>>> response.css('title::text').get()
|
||||
'Quotes to Scrape'
|
||||
|
||||
As an alternative, you could've written::
|
||||
|
||||
>>> response.css('title::text')[0].extract()
|
||||
>>> response.css('title::text')[0].get()
|
||||
'Quotes to Scrape'
|
||||
|
||||
However, using ``.extract_first()`` avoids an ``IndexError`` and returns
|
||||
``None`` when it doesn't find any element matching the selection.
|
||||
However, using ``.get()`` directly on a :class:`~scrapy.selector.SelectorList`
|
||||
instance avoids an ``IndexError`` and returns ``None`` when it doesn't
|
||||
find any element matching the selection.
|
||||
|
||||
There's a lesson here: for most scraping code, you want it to be resilient to
|
||||
errors due to things not being found on a page, so that even if some parts fail
|
||||
to be scraped, you can at least get **some** data.
|
||||
|
||||
Besides the :meth:`~scrapy.selector.Selector.extract` and
|
||||
:meth:`~scrapy.selector.SelectorList.extract_first` methods, you can also use
|
||||
the :meth:`~scrapy.selector.Selector.re` method to extract using `regular
|
||||
expressions`::
|
||||
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
|
||||
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
|
||||
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
|
||||
expressions`_::
|
||||
|
||||
>>> response.css('title::text').re(r'Quotes.*')
|
||||
['Quotes to Scrape']
|
||||
|
|
@ -298,7 +296,8 @@ expressions`::
|
|||
|
||||
In order to find the proper CSS selectors to use, you might find useful opening
|
||||
the response page from the shell in your web browser using ``view(response)``.
|
||||
You can use your browser developer tools (see section about :ref:`topics-developer-tools`).
|
||||
You can use your browser developer tools to inspect the HTML and come up
|
||||
with a selector (see section about :ref:`topics-developer-tools`).
|
||||
|
||||
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
|
||||
visually selected elements, which works in many browsers.
|
||||
|
|
@ -314,7 +313,7 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions::
|
|||
|
||||
>>> response.xpath('//title')
|
||||
[<Selector xpath='//title' data='<title>Quotes to Scrape</title>'>]
|
||||
>>> response.xpath('//title/text()').extract_first()
|
||||
>>> response.xpath('//title/text()').get()
|
||||
'Quotes to Scrape'
|
||||
|
||||
XPath expressions are very powerful, and are the foundation of Scrapy
|
||||
|
|
@ -383,17 +382,17 @@ variable, so that we can run our CSS selectors directly on a particular quote::
|
|||
Now, let's extract ``title``, ``author`` and the ``tags`` from that quote
|
||||
using the ``quote`` object we just created::
|
||||
|
||||
>>> title = quote.css("span.text::text").extract_first()
|
||||
>>> title = quote.css("span.text::text").get()
|
||||
>>> title
|
||||
'“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'
|
||||
>>> author = quote.css("small.author::text").extract_first()
|
||||
>>> author = quote.css("small.author::text").get()
|
||||
>>> author
|
||||
'Albert Einstein'
|
||||
|
||||
Given that the tags are a list of strings, we can use the ``.extract()`` method
|
||||
Given that the tags are a list of strings, we can use the ``.getall()`` method
|
||||
to get all of them::
|
||||
|
||||
>>> tags = quote.css("div.tags a.tag::text").extract()
|
||||
>>> tags = quote.css("div.tags a.tag::text").getall()
|
||||
>>> tags
|
||||
['change', 'deep-thoughts', 'thinking', 'world']
|
||||
|
||||
|
|
@ -401,9 +400,9 @@ Having figured out how to extract each bit, we can now iterate over all the
|
|||
quotes elements and put them together into a Python dictionary::
|
||||
|
||||
>>> for quote in response.css("div.quote"):
|
||||
... text = quote.css("span.text::text").extract_first()
|
||||
... author = quote.css("small.author::text").extract_first()
|
||||
... tags = quote.css("div.tags a.tag::text").extract()
|
||||
... text = quote.css("span.text::text").get()
|
||||
... author = quote.css("small.author::text").get()
|
||||
... tags = quote.css("div.tags a.tag::text").getall()
|
||||
... print(dict(text=text, author=author, tags=tags))
|
||||
{'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
|
||||
{'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
|
||||
|
|
@ -434,9 +433,9 @@ in the callback, as you can see below::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('small.author::text').extract_first(),
|
||||
'tags': quote.css('div.tags a.tag::text').extract(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.css('small.author::text').get(),
|
||||
'tags': quote.css('div.tags a.tag::text').getall(),
|
||||
}
|
||||
|
||||
If you run this spider, it will output the extracted data with the log::
|
||||
|
|
@ -508,16 +507,22 @@ markup:
|
|||
|
||||
We can try extracting it in the shell::
|
||||
|
||||
>>> response.css('li.next a').extract_first()
|
||||
>>> response.css('li.next a').get()
|
||||
'<a href="/page/2/">Next <span aria-hidden="true">→</span></a>'
|
||||
|
||||
This gets the anchor element, but we want the attribute ``href``. For that,
|
||||
Scrapy supports a CSS extension that let's you select the attribute contents,
|
||||
like this::
|
||||
|
||||
>>> response.css('li.next a::attr(href)').extract_first()
|
||||
>>> response.css('li.next a::attr(href)').get()
|
||||
'/page/2/'
|
||||
|
||||
There is also an ``attrib`` property available
|
||||
(see :ref:`selecting-attributes` for more)::
|
||||
|
||||
>>> response.css('li.next a').attrib['href']
|
||||
'/page/2'
|
||||
|
||||
Let's see now our spider modified to recursively follow the link to the next
|
||||
page, extracting data from it::
|
||||
|
||||
|
|
@ -533,12 +538,12 @@ page, extracting data from it::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('small.author::text').extract_first(),
|
||||
'tags': quote.css('div.tags a.tag::text').extract(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.css('small.author::text').get(),
|
||||
'tags': quote.css('div.tags a.tag::text').getall(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr(href)').extract_first()
|
||||
next_page = response.css('li.next a::attr(href)').get()
|
||||
if next_page is not None:
|
||||
next_page = response.urljoin(next_page)
|
||||
yield scrapy.Request(next_page, callback=self.parse)
|
||||
|
|
@ -584,12 +589,12 @@ As a shortcut for creating Request objects you can use
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('span small::text').extract_first(),
|
||||
'tags': quote.css('div.tags a.tag::text').extract(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.css('span small::text').get(),
|
||||
'tags': quote.css('div.tags a.tag::text').getall(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr(href)').extract_first()
|
||||
next_page = response.css('li.next a::attr(href)').get()
|
||||
if next_page is not None:
|
||||
yield response.follow(next_page, callback=self.parse)
|
||||
|
||||
|
|
@ -641,7 +646,7 @@ this time for scraping author information::
|
|||
|
||||
def parse_author(self, response):
|
||||
def extract_with_css(query):
|
||||
return response.css(query).extract_first().strip()
|
||||
return response.css(query).get(default='').strip()
|
||||
|
||||
yield {
|
||||
'name': extract_with_css('h3.author-title::text'),
|
||||
|
|
@ -710,11 +715,11 @@ with a specific tag, building the URL based on the argument::
|
|||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('small.author::text').extract_first(),
|
||||
'text': quote.css('span.text::text').get(),
|
||||
'author': quote.css('small.author::text').get(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr(href)').extract_first()
|
||||
next_page = response.css('li.next a::attr(href)').get()
|
||||
if next_page is not None:
|
||||
yield response.follow(next_page, self.parse)
|
||||
|
||||
|
|
@ -738,4 +743,3 @@ modeling the scraped data. If you prefer to play with an example project, check
|
|||
the :ref:`intro-examples` section.
|
||||
|
||||
.. _JSON: https://en.wikipedia.org/wiki/JSON
|
||||
.. _dirbot: https://github.com/scrapy/dirbot
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment
|
|||
variables. Currently these are:
|
||||
|
||||
* ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`)
|
||||
* ``SCRAPY_PROJECT``
|
||||
* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`)
|
||||
* ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`)
|
||||
|
||||
.. _topics-project-structure:
|
||||
|
|
@ -71,6 +71,33 @@ the project settings. Here is an example::
|
|||
[settings]
|
||||
default = myproject.settings
|
||||
|
||||
.. _topics-project-envvar:
|
||||
|
||||
Sharing the root directory between projects
|
||||
===========================================
|
||||
|
||||
A project root directory, the one that contains the ``scrapy.cfg``, may be
|
||||
shared by multiple Scrapy projects, each with its own settings module.
|
||||
|
||||
In that case, you must define one or more aliases for those settings modules
|
||||
under ``[settings]`` in your ``scrapy.cfg`` file::
|
||||
|
||||
[settings]
|
||||
default = myproject1.settings
|
||||
project1 = myproject1.settings
|
||||
project2 = myproject2.settings
|
||||
|
||||
By default, the ``scrapy`` command-line tool will use the ``default`` settings.
|
||||
Use the ``SCRAPY_PROJECT`` environment variable to specify a different project
|
||||
for ``scrapy`` to use::
|
||||
|
||||
$ scrapy settings --get BOT_NAME
|
||||
Project 1 Bot
|
||||
$ export SCRAPY_PROJECT=project2
|
||||
$ scrapy settings --get BOT_NAME
|
||||
Project 2 Bot
|
||||
|
||||
|
||||
Using the ``scrapy`` tool
|
||||
=========================
|
||||
|
||||
|
|
@ -458,9 +485,9 @@ Usage example::
|
|||
|
||||
>>> STATUS DEPTH LEVEL 1 <<<
|
||||
# Scraped Items ------------------------------------------------------------
|
||||
[{'name': u'Example item',
|
||||
'category': u'Furniture',
|
||||
'length': u'12 cm'}]
|
||||
[{'name': 'Example item',
|
||||
'category': 'Furniture',
|
||||
'length': '12 cm'}]
|
||||
|
||||
# Requests -----------------------------------------------------------------
|
||||
[]
|
||||
|
|
|
|||
|
|
@ -86,8 +86,11 @@ override three methods:
|
|||
.. method:: Contract.adjust_request_args(args)
|
||||
|
||||
This receives a ``dict`` as an argument containing default arguments
|
||||
for :class:`~scrapy.http.Request` object. Must return the same or a
|
||||
modified version of it.
|
||||
for request object. :class:`~scrapy.http.Request` is used by default,
|
||||
but this can be changed with the ``request_cls`` attribute.
|
||||
If multiple contracts in chain have this attribute defined, the last one is used.
|
||||
|
||||
Must return the same or a modified version of it.
|
||||
|
||||
.. method:: Contract.pre_process(response)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,11 +18,13 @@ Consider the following scrapy spider below::
|
|||
)
|
||||
|
||||
def parse(self, response):
|
||||
# collect `item_urls`
|
||||
# <processing code not shown>
|
||||
# collect `item_urls`
|
||||
for item_url in item_urls:
|
||||
yield scrapy.Request(item_url, self.parse_item)
|
||||
|
||||
def parse_item(self, response):
|
||||
# <processing code not shown>
|
||||
item = MyItem()
|
||||
# populate `item` fields
|
||||
# and extract item_details_url
|
||||
|
|
|
|||
|
|
@ -86,7 +86,7 @@ Creating items
|
|||
::
|
||||
|
||||
>>> product = Product(name='Desktop PC', price=1000)
|
||||
>>> print product
|
||||
>>> print(product)
|
||||
Product(name='Desktop PC', price=1000)
|
||||
|
||||
Getting field values
|
||||
|
|
@ -161,11 +161,11 @@ Other common tasks
|
|||
Copying items::
|
||||
|
||||
>>> product2 = Product(product)
|
||||
>>> print product2
|
||||
>>> print(product2)
|
||||
Product(name='Desktop PC', price=1000)
|
||||
|
||||
>>> product3 = product2.copy()
|
||||
>>> print product3
|
||||
>>> print(product3)
|
||||
Product(name='Desktop PC', price=1000)
|
||||
|
||||
Creating dicts from items::
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ So, for example, this won't work::
|
|||
return scrapy.Request('http://www.example.com', callback=lambda r: self.other_callback(r, somearg))
|
||||
|
||||
def other_callback(self, response, somearg):
|
||||
print "the argument passed is:", somearg
|
||||
print("the argument passed is: %s" % somearg)
|
||||
|
||||
But this will::
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ But this will::
|
|||
|
||||
def other_callback(self, response):
|
||||
somearg = response.meta['somearg']
|
||||
print "the argument passed is:", somearg
|
||||
print("the argument passed is: %s" % somearg)
|
||||
|
||||
If you wish to log the requests that couldn't be serialized, you can set the
|
||||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||
|
|
|
|||
|
|
@ -678,10 +678,10 @@ Here is a list of all built-in processors:
|
|||
>>> from scrapy.loader.processors import Join
|
||||
>>> proc = Join()
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
u'one two three'
|
||||
'one two three'
|
||||
>>> proc = Join('<br>')
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
u'one<br>two<br>three'
|
||||
'one<br>two<br>three'
|
||||
|
||||
.. class:: Compose(\*functions, \**default_loader_context)
|
||||
|
||||
|
|
@ -744,9 +744,9 @@ Here is a list of all built-in processors:
|
|||
... return None if x == 'world' else x
|
||||
...
|
||||
>>> from scrapy.loader.processors import MapCompose
|
||||
>>> proc = MapCompose(filter_world, unicode.upper)
|
||||
>>> proc([u'hello', u'world', u'this', u'is', u'scrapy'])
|
||||
[u'HELLO, u'THIS', u'IS', u'SCRAPY']
|
||||
>>> proc = MapCompose(filter_world, str.upper)
|
||||
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
|
||||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
constructor keyword arguments are used as default context values. See
|
||||
|
|
@ -772,7 +772,7 @@ Here is a list of all built-in processors:
|
|||
>>> import json
|
||||
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
|
||||
>>> proc_single_json_str('{"foo": "bar"}')
|
||||
u'bar'
|
||||
'bar'
|
||||
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
|
||||
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
|
||||
[u'bar']
|
||||
['bar']
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -871,7 +871,7 @@ LOG_STDOUT
|
|||
Default: ``False``
|
||||
|
||||
If ``True``, all standard output (and error) of your process will be redirected
|
||||
to the log. For example if you ``print 'hello'`` it will appear in the Scrapy
|
||||
to the log. For example if you ``print('hello')`` it will appear in the Scrapy
|
||||
log.
|
||||
|
||||
.. setting:: LOG_SHORT_NAMES
|
||||
|
|
|
|||
|
|
@ -179,13 +179,13 @@ all start with the ``[s]`` prefix)::
|
|||
|
||||
After that, we can start playing with the objects::
|
||||
|
||||
>>> response.xpath('//title/text()').extract_first()
|
||||
>>> response.xpath('//title/text()').get()
|
||||
'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework'
|
||||
|
||||
>>> fetch("https://reddit.com")
|
||||
|
||||
>>> response.xpath('//title/text()').extract()
|
||||
['reddit: the front page of the internet']
|
||||
>>> response.xpath('//title/text()').get()
|
||||
'reddit: the front page of the internet'
|
||||
|
||||
>>> request = request.replace(method="POST")
|
||||
|
||||
|
|
|
|||
|
|
@ -279,6 +279,22 @@ request_dropped
|
|||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
request_reached_downloader
|
||||
---------------------------
|
||||
|
||||
.. signal:: request_reached_downloader
|
||||
.. function:: request_reached_downloader(request, spider)
|
||||
|
||||
Sent when a :class:`~scrapy.http.Request` reached downloader.
|
||||
|
||||
The signal does not support returning deferreds from their handlers.
|
||||
|
||||
:param request: the request that reached downloader
|
||||
:type request: :class:`~scrapy.http.Request` object
|
||||
|
||||
:param spider: the spider that yielded the request
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
response_received
|
||||
-----------------
|
||||
|
||||
|
|
|
|||
|
|
@ -229,11 +229,11 @@ Return multiple Requests and items from a single callback::
|
|||
]
|
||||
|
||||
def parse(self, response):
|
||||
for h3 in response.xpath('//h3').extract():
|
||||
for h3 in response.xpath('//h3').getall():
|
||||
yield {"title": h3}
|
||||
|
||||
for url in response.xpath('//a/@href').extract():
|
||||
yield scrapy.Request(url, callback=self.parse)
|
||||
for href in response.xpath('//a/@href').getall():
|
||||
yield scrapy.Request(response.urljoin(href), self.parse)
|
||||
|
||||
Instead of :attr:`~.start_urls` you can use :meth:`~.start_requests` directly;
|
||||
to give data more structure you can use :ref:`topics-items`::
|
||||
|
|
@ -251,11 +251,11 @@ to give data more structure you can use :ref:`topics-items`::
|
|||
yield scrapy.Request('http://www.example.com/3.html', self.parse)
|
||||
|
||||
def parse(self, response):
|
||||
for h3 in response.xpath('//h3').extract():
|
||||
for h3 in response.xpath('//h3').getall():
|
||||
yield MyItem(title=h3)
|
||||
|
||||
for url in response.xpath('//a/@href').extract():
|
||||
yield scrapy.Request(url, callback=self.parse)
|
||||
for href in response.xpath('//a/@href').getall():
|
||||
yield scrapy.Request(response.urljoin(href), self.parse)
|
||||
|
||||
.. _spiderargs:
|
||||
|
||||
|
|
@ -434,8 +434,8 @@ Let's now take a look at an example CrawlSpider with rules::
|
|||
self.logger.info('Hi, this is an item page! %s', response.url)
|
||||
item = scrapy.Item()
|
||||
item['id'] = response.xpath('//td[@id="item_id"]/text()').re(r'ID: (\d+)')
|
||||
item['name'] = response.xpath('//td[@id="item_name"]/text()').extract()
|
||||
item['description'] = response.xpath('//td[@id="item_description"]/text()').extract()
|
||||
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
|
||||
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
|
||||
return item
|
||||
|
||||
|
||||
|
|
@ -545,12 +545,12 @@ These spiders are pretty easy to use, let's have a look at one example::
|
|||
itertag = 'item'
|
||||
|
||||
def parse_node(self, response, node):
|
||||
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.extract()))
|
||||
self.logger.info('Hi, this is a <%s> node!: %s', self.itertag, ''.join(node.getall()))
|
||||
|
||||
item = TestItem()
|
||||
item['id'] = node.xpath('@id').extract()
|
||||
item['name'] = node.xpath('name').extract()
|
||||
item['description'] = node.xpath('description').extract()
|
||||
item['id'] = node.xpath('@id').get()
|
||||
item['name'] = node.xpath('name').get()
|
||||
item['description'] = node.xpath('description').get()
|
||||
return item
|
||||
|
||||
Basically what we did up there was to create a spider that downloads a feed from
|
||||
|
|
|
|||
|
|
@ -6,5 +6,5 @@ queuelib
|
|||
w3lib>=1.17.0
|
||||
six>=1.5.2
|
||||
PyDispatcher>=2.0.5
|
||||
parsel>=1.4
|
||||
parsel>=1.5
|
||||
service_identity
|
||||
|
|
|
|||
|
|
@ -6,5 +6,5 @@ queuelib>=1.1.1
|
|||
w3lib>=1.17.0
|
||||
six>=1.5.2
|
||||
PyDispatcher>=2.0.5
|
||||
parsel>=1.4
|
||||
parsel>=1.5
|
||||
service_identity
|
||||
|
|
|
|||
|
|
@ -42,23 +42,38 @@ class ContractsManager(object):
|
|||
requests = []
|
||||
for method in self.tested_methods_from_spidercls(type(spider)):
|
||||
bound_method = spider.__getattribute__(method)
|
||||
requests.append(self.from_method(bound_method, results))
|
||||
try:
|
||||
requests.append(self.from_method(bound_method, results))
|
||||
except Exception:
|
||||
case = _create_testcase(bound_method, 'contract')
|
||||
results.addError(case, sys.exc_info())
|
||||
|
||||
return requests
|
||||
|
||||
def from_method(self, method, results):
|
||||
contracts = self.extract_contracts(method)
|
||||
if contracts:
|
||||
request_cls = Request
|
||||
for contract in contracts:
|
||||
if contract.request_cls is not None:
|
||||
request_cls = contract.request_cls
|
||||
|
||||
# calculate request args
|
||||
args, kwargs = get_spec(Request.__init__)
|
||||
args, kwargs = get_spec(request_cls.__init__)
|
||||
|
||||
# Don't filter requests to allow
|
||||
# testing different callbacks on the same URL.
|
||||
kwargs['dont_filter'] = True
|
||||
kwargs['callback'] = method
|
||||
|
||||
for contract in contracts:
|
||||
kwargs = contract.adjust_request_args(kwargs)
|
||||
|
||||
# create and prepare request
|
||||
args.remove('self')
|
||||
|
||||
# check if all positional arguments are defined in kwargs
|
||||
if set(args).issubset(set(kwargs)):
|
||||
request = Request(**kwargs)
|
||||
request = request_cls(**kwargs)
|
||||
|
||||
# execute pre and post hooks in order
|
||||
for contract in reversed(contracts):
|
||||
|
|
@ -94,6 +109,7 @@ class ContractsManager(object):
|
|||
|
||||
class Contract(object):
|
||||
""" Abstract class for contracts """
|
||||
request_cls = None
|
||||
|
||||
def __init__(self, method, *args):
|
||||
self.testcase_pre = _create_testcase(method, '@%s pre-hook' % self.name)
|
||||
|
|
|
|||
|
|
@ -129,6 +129,9 @@ class Downloader(object):
|
|||
return response
|
||||
|
||||
slot.active.add(request)
|
||||
self.signals.send_catch_log(signal=signals.request_reached_downloader,
|
||||
request=request,
|
||||
spider=spider)
|
||||
deferred = defer.Deferred().addBoth(_deactivate)
|
||||
slot.queue.append((request, deferred))
|
||||
self._process_queue(spider, slot)
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
|
|||
base_url = get_base_url(response)
|
||||
body = u''.join(f
|
||||
for x in self.restrict_xpaths
|
||||
for f in response.xpath(x).extract()
|
||||
for f in response.xpath(x).getall()
|
||||
).encode(response.encoding, errors='xmlcharrefreplace')
|
||||
else:
|
||||
body = response.body
|
||||
|
|
|
|||
|
|
@ -181,7 +181,7 @@ class ItemLoader(object):
|
|||
def _get_xpathvalues(self, xpaths, **kw):
|
||||
self._check_selector_method()
|
||||
xpaths = arg_to_iter(xpaths)
|
||||
return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths)
|
||||
return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
|
||||
|
||||
def add_css(self, field_name, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
|
|
@ -198,6 +198,6 @@ class ItemLoader(object):
|
|||
def _get_cssvalues(self, csss, **kw):
|
||||
self._check_selector_method()
|
||||
csss = arg_to_iter(csss)
|
||||
return flatten(self.selector.css(css).extract() for css in csss)
|
||||
return flatten(self.selector.css(css).getall() for css in csss)
|
||||
|
||||
XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)
|
||||
|
|
|
|||
|
|
@ -27,6 +27,10 @@ def _response_from_text(text, st):
|
|||
|
||||
|
||||
class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
|
||||
"""
|
||||
The :class:`SelectorList` class is a subclass of the builtin ``list``
|
||||
class, which provides a few additional methods.
|
||||
"""
|
||||
@deprecated(use_instead='.extract()')
|
||||
def extract_unquoted(self):
|
||||
return [x.extract_unquoted() for x in self]
|
||||
|
|
@ -41,6 +45,35 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
|
|||
|
||||
|
||||
class Selector(_ParselSelector, object_ref):
|
||||
"""
|
||||
An instance of :class:`Selector` is a wrapper over response to select
|
||||
certain parts of its content.
|
||||
|
||||
``response`` is an :class:`~scrapy.http.HtmlResponse` or an
|
||||
:class:`~scrapy.http.XmlResponse` object that will be used for selecting
|
||||
and extracting data.
|
||||
|
||||
``text`` is a unicode string or utf-8 encoded text for cases when a
|
||||
``response`` isn't available. Using ``text`` and ``response`` together is
|
||||
undefined behavior.
|
||||
|
||||
``type`` defines the selector type, it can be ``"html"``, ``"xml"``
|
||||
or ``None`` (default).
|
||||
|
||||
If ``type`` is ``None``, the selector automatically chooses the best type
|
||||
based on ``response`` type (see below), or defaults to ``"html"`` in case it
|
||||
is used together with ``text``.
|
||||
|
||||
If ``type`` is ``None`` and a ``response`` is passed, the selector type is
|
||||
inferred from the response type as follows:
|
||||
|
||||
* ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
|
||||
* ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
|
||||
* ``"html"`` for anything else
|
||||
|
||||
Otherwise, if ``type`` is set, the selector type will be forced and no
|
||||
detection will occur.
|
||||
"""
|
||||
|
||||
__slots__ = ['response']
|
||||
selectorlist_cls = SelectorList
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ spider_closed = object()
|
|||
spider_error = object()
|
||||
request_scheduled = object()
|
||||
request_dropped = object()
|
||||
request_reached_downloader = object()
|
||||
response_received = object()
|
||||
response_downloaded = object()
|
||||
item_scraped = object()
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ class $classname(CrawlSpider):
|
|||
)
|
||||
|
||||
def parse_item(self, response):
|
||||
i = {}
|
||||
#i['domain_id'] = response.xpath('//input[@id="sid"]/@value').extract()
|
||||
#i['name'] = response.xpath('//div[@id="name"]').extract()
|
||||
#i['description'] = response.xpath('//div[@id="description"]').extract()
|
||||
return i
|
||||
item = {}
|
||||
#item['domain_id'] = response.xpath('//input[@id="sid"]/@value').get()
|
||||
#item['name'] = response.xpath('//div[@id="name"]').get()
|
||||
#item['description'] = response.xpath('//div[@id="description"]').get()
|
||||
return item
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ class $classname(XMLFeedSpider):
|
|||
itertag = 'item' # change it accordingly
|
||||
|
||||
def parse_node(self, response, selector):
|
||||
i = {}
|
||||
#i['url'] = selector.select('url').extract()
|
||||
#i['name'] = selector.select('name').extract()
|
||||
#i['description'] = selector.select('description').extract()
|
||||
return i
|
||||
item = {}
|
||||
#item['url'] = selector.select('url').get()
|
||||
#item['name'] = selector.select('name').get()
|
||||
#item['description'] = selector.select('description').get()
|
||||
return item
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -71,7 +71,7 @@ setup(
|
|||
'pyOpenSSL',
|
||||
'cssselect>=0.9',
|
||||
'six>=1.5.2',
|
||||
'parsel>=1.4',
|
||||
'parsel>=1.5',
|
||||
'PyDispatcher>=2.0.5',
|
||||
'service_identity',
|
||||
],
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_selector_html(self):
|
||||
xpath = 'response.xpath("//p[@class=\'one\']/text()").extract()[0]'
|
||||
xpath = 'response.xpath("//p[@class=\'one\']/text()").get()'
|
||||
_, out, _ = yield self.execute([self.url('/html'), '-c', xpath])
|
||||
self.assertEqual(out.strip(), b'Works')
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,23 @@
|
|||
from unittest import TextTestResult
|
||||
|
||||
from six import get_unbound_function
|
||||
from twisted.internet import defer
|
||||
from twisted.python import failure
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy import FormRequest
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.spidermiddlewares.httperror import HttpError
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.http import Request
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.contracts import ContractsManager
|
||||
from scrapy.contracts import ContractsManager, Contract
|
||||
from scrapy.contracts.default import (
|
||||
UrlContract,
|
||||
ReturnsContract,
|
||||
ScrapesContract,
|
||||
)
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
|
||||
class TestItem(Item):
|
||||
|
|
@ -24,6 +29,30 @@ class ResponseMock(object):
|
|||
url = 'http://scrapy.org'
|
||||
|
||||
|
||||
class CustomSuccessContract(Contract):
|
||||
name = 'custom_success_contract'
|
||||
|
||||
def adjust_request_args(self, args):
|
||||
args['url'] = 'http://scrapy.org'
|
||||
return args
|
||||
|
||||
|
||||
class CustomFailContract(Contract):
|
||||
name = 'custom_fail_contract'
|
||||
|
||||
def adjust_request_args(self, args):
|
||||
raise TypeError('Error in adjust_request_args')
|
||||
|
||||
|
||||
class CustomFormContract(Contract):
|
||||
name = 'custom_form'
|
||||
request_cls = FormRequest
|
||||
|
||||
def adjust_request_args(self, args):
|
||||
args['formdata'] = {'name': 'scrapy'}
|
||||
return args
|
||||
|
||||
|
||||
class TestSpider(Spider):
|
||||
name = 'demo_spider'
|
||||
|
||||
|
|
@ -100,13 +129,47 @@ class TestSpider(Spider):
|
|||
"""
|
||||
pass
|
||||
|
||||
def custom_form(self, response):
|
||||
"""
|
||||
@url http://scrapy.org
|
||||
@custom_form
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CustomContractSuccessSpider(Spider):
|
||||
name = 'custom_contract_success_spider'
|
||||
|
||||
def parse(self, response):
|
||||
"""
|
||||
@custom_success_contract
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class CustomContractFailSpider(Spider):
|
||||
name = 'custom_contract_fail_spider'
|
||||
|
||||
def parse(self, response):
|
||||
"""
|
||||
@custom_fail_contract
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class InheritsTestSpider(TestSpider):
|
||||
name = 'inherits_demo_spider'
|
||||
|
||||
|
||||
class ContractsManagerTest(unittest.TestCase):
|
||||
contracts = [UrlContract, ReturnsContract, ScrapesContract]
|
||||
contracts = [
|
||||
UrlContract,
|
||||
ReturnsContract,
|
||||
ScrapesContract,
|
||||
CustomFormContract,
|
||||
CustomSuccessContract,
|
||||
CustomFailContract,
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
self.conman = ContractsManager(self.contracts)
|
||||
|
|
@ -120,6 +183,9 @@ class ContractsManagerTest(unittest.TestCase):
|
|||
self.assertTrue(self.results.failures)
|
||||
self.assertFalse(self.results.errors)
|
||||
|
||||
def should_error(self):
|
||||
self.assertTrue(self.results.errors)
|
||||
|
||||
def test_contracts(self):
|
||||
spider = TestSpider()
|
||||
|
||||
|
|
@ -181,17 +247,22 @@ class ContractsManagerTest(unittest.TestCase):
|
|||
self.should_succeed()
|
||||
|
||||
# scrapes_item_fail
|
||||
request = self.conman.from_method(spider.scrapes_item_fail,
|
||||
self.results)
|
||||
request = self.conman.from_method(spider.scrapes_item_fail, self.results)
|
||||
request.callback(response)
|
||||
self.should_fail()
|
||||
|
||||
# scrapes_dict_item_fail
|
||||
request = self.conman.from_method(spider.scrapes_dict_item_fail,
|
||||
self.results)
|
||||
request = self.conman.from_method(spider.scrapes_dict_item_fail, self.results)
|
||||
request.callback(response)
|
||||
self.should_fail()
|
||||
|
||||
def test_custom_contracts(self):
|
||||
self.conman.from_spider(CustomContractSuccessSpider(), self.results)
|
||||
self.should_succeed()
|
||||
|
||||
self.conman.from_spider(CustomContractFailSpider(), self.results)
|
||||
self.should_error()
|
||||
|
||||
def test_errback(self):
|
||||
spider = TestSpider()
|
||||
response = ResponseMock()
|
||||
|
|
@ -207,6 +278,44 @@ class ContractsManagerTest(unittest.TestCase):
|
|||
self.assertFalse(self.results.failures)
|
||||
self.assertTrue(self.results.errors)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_same_url(self):
|
||||
|
||||
class TestSameUrlSpider(Spider):
|
||||
name = 'test_same_url'
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(TestSameUrlSpider, self).__init__(*args, **kwargs)
|
||||
self.visited = 0
|
||||
|
||||
def start_requests(s):
|
||||
return self.conman.from_spider(s, self.results)
|
||||
|
||||
def parse_first(self, response):
|
||||
self.visited += 1
|
||||
return TestItem()
|
||||
|
||||
def parse_second(self, response):
|
||||
self.visited += 1
|
||||
return TestItem()
|
||||
|
||||
with MockServer() as mockserver:
|
||||
contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
|
||||
|
||||
get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
|
||||
get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
|
||||
|
||||
crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
|
||||
yield crawler.crawl()
|
||||
|
||||
self.assertEqual(crawler.spider.visited, 2)
|
||||
|
||||
def test_form_contract(self):
|
||||
spider = TestSpider()
|
||||
request = self.conman.from_method(spider.custom_form, self.results)
|
||||
self.assertEqual(request.method, 'POST')
|
||||
self.assertIsInstance(request, FormRequest)
|
||||
|
||||
def test_inherited_contracts(self):
|
||||
spider = InheritsTestSpider()
|
||||
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ class CrawlerRun(object):
|
|||
self.respplug = []
|
||||
self.reqplug = []
|
||||
self.reqdropped = []
|
||||
self.reqreached = []
|
||||
self.itemerror = []
|
||||
self.itemresp = []
|
||||
self.signals_catched = {}
|
||||
|
|
@ -124,6 +125,7 @@ class CrawlerRun(object):
|
|||
self.crawler.signals.connect(self.item_error, signals.item_error)
|
||||
self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
|
||||
self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
|
||||
self.crawler.signals.connect(self.request_reached, signals.request_reached_downloader)
|
||||
self.crawler.signals.connect(self.response_downloaded, signals.response_downloaded)
|
||||
self.crawler.crawl(start_urls=start_urls)
|
||||
self.spider = self.crawler.spider
|
||||
|
|
@ -155,6 +157,9 @@ class CrawlerRun(object):
|
|||
def request_scheduled(self, request, spider):
|
||||
self.reqplug.append((request, spider))
|
||||
|
||||
def request_reached(self, request, spider):
|
||||
self.reqreached.append((request, spider))
|
||||
|
||||
def request_dropped(self, request, spider):
|
||||
self.reqdropped.append((request, spider))
|
||||
|
||||
|
|
@ -212,6 +217,8 @@ class EngineTest(unittest.TestCase):
|
|||
responses_count = len(self.run.respplug)
|
||||
self.assertEqual(scheduled_requests_count,
|
||||
dropped_requests_count + responses_count)
|
||||
self.assertEqual(len(self.run.reqreached),
|
||||
responses_count)
|
||||
|
||||
def _assert_dropped_requests(self):
|
||||
self.assertEqual(len(self.run.reqdropped), 1)
|
||||
|
|
@ -219,6 +226,7 @@ class EngineTest(unittest.TestCase):
|
|||
def _assert_downloaded_responses(self):
|
||||
# response tests
|
||||
self.assertEqual(8, len(self.run.respplug))
|
||||
self.assertEqual(8, len(self.run.reqreached))
|
||||
|
||||
for response, _ in self.run.respplug:
|
||||
if self.run.getpath(response.url) == '/item999.html':
|
||||
|
|
|
|||
|
|
@ -336,11 +336,11 @@ class TextResponseTest(BaseResponseTest):
|
|||
self.assertIs(response.selector.response, response)
|
||||
|
||||
self.assertEqual(
|
||||
response.selector.xpath("//title/text()").extract(),
|
||||
response.selector.xpath("//title/text()").getall(),
|
||||
[u'Some page']
|
||||
)
|
||||
self.assertEqual(
|
||||
response.selector.css("title::text").extract(),
|
||||
response.selector.css("title::text").getall(),
|
||||
[u'Some page']
|
||||
)
|
||||
self.assertEqual(
|
||||
|
|
@ -353,12 +353,12 @@ class TextResponseTest(BaseResponseTest):
|
|||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("//title/text()").extract(),
|
||||
response.selector.xpath("//title/text()").extract(),
|
||||
response.xpath("//title/text()").getall(),
|
||||
response.selector.xpath("//title/text()").getall(),
|
||||
)
|
||||
self.assertEqual(
|
||||
response.css("title::text").extract(),
|
||||
response.selector.css("title::text").extract(),
|
||||
response.css("title::text").getall(),
|
||||
response.selector.css("title::text").getall(),
|
||||
)
|
||||
|
||||
def test_selector_shortcuts_kwargs(self):
|
||||
|
|
@ -366,13 +366,13 @@ class TextResponseTest(BaseResponseTest):
|
|||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
|
||||
response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
|
||||
response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").getall(),
|
||||
response.xpath("normalize-space(//p[@class=\"content\"])").getall(),
|
||||
)
|
||||
self.assertEqual(
|
||||
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
|
||||
pclass="content", pcount=1).extract(),
|
||||
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
|
||||
pclass="content", pcount=1).getall(),
|
||||
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").getall(),
|
||||
)
|
||||
|
||||
def test_urljoin_with_base_url(self):
|
||||
|
|
@ -562,7 +562,7 @@ class XmlResponseTest(TextResponseTest):
|
|||
self.assertIs(response.selector.response, response)
|
||||
|
||||
self.assertEqual(
|
||||
response.selector.xpath("//elem/text()").extract(),
|
||||
response.selector.xpath("//elem/text()").getall(),
|
||||
[u'value']
|
||||
)
|
||||
|
||||
|
|
@ -571,8 +571,8 @@ class XmlResponseTest(TextResponseTest):
|
|||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("//elem/text()").extract(),
|
||||
response.selector.xpath("//elem/text()").extract(),
|
||||
response.xpath("//elem/text()").getall(),
|
||||
response.selector.xpath("//elem/text()").getall(),
|
||||
)
|
||||
|
||||
def test_selector_shortcuts_kwargs(self):
|
||||
|
|
@ -583,12 +583,12 @@ class XmlResponseTest(TextResponseTest):
|
|||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
|
||||
response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
|
||||
response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
|
||||
response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).getall(),
|
||||
)
|
||||
|
||||
response.selector.register_namespace('s2', 'http://scrapy.org')
|
||||
self.assertEqual(
|
||||
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
|
||||
response.selector.xpath("//s2:elem/text()").extract(),
|
||||
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).getall(),
|
||||
response.selector.xpath("//s2:elem/text()").getall(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -634,7 +634,7 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl = l.nested_xpath("//header")
|
||||
nl.add_xpath('name', 'div/text()')
|
||||
nl.add_css('name_div', '#id')
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
|
|
@ -649,7 +649,7 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
nl = l.nested_css("header")
|
||||
nl.add_xpath('name', 'div/text()')
|
||||
nl.add_css('name_div', '#id')
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').extract())
|
||||
nl.add_value('name_value', nl.selector.xpath('div[@id = "id"]/text()').getall())
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class MediaDownloadSpider(SimpleSpider):
|
|||
for href in response.xpath('''
|
||||
//table[thead/tr/th="Filename"]
|
||||
/tbody//a/@href
|
||||
''').extract()],
|
||||
''').getall()],
|
||||
}
|
||||
yield item
|
||||
|
||||
|
|
|
|||
|
|
@ -20,17 +20,17 @@ class SelectorTestCase(unittest.TestCase):
|
|||
for x in xl:
|
||||
assert isinstance(x, Selector)
|
||||
|
||||
self.assertEqual(sel.xpath('//input').extract(),
|
||||
[x.extract() for x in sel.xpath('//input')])
|
||||
self.assertEqual(sel.xpath('//input').getall(),
|
||||
[x.get() for x in sel.xpath('//input')])
|
||||
|
||||
self.assertEqual([x.extract() for x in sel.xpath("//input[@name='a']/@name")],
|
||||
self.assertEqual([x.get() for x in sel.xpath("//input[@name='a']/@name")],
|
||||
[u'a'])
|
||||
self.assertEqual([x.extract() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
self.assertEqual([x.get() for x in sel.xpath("number(concat(//input[@name='a']/@value, //input[@name='b']/@value))")],
|
||||
[u'12.0'])
|
||||
|
||||
self.assertEqual(sel.xpath("concat('xpath', 'rules')").extract(),
|
||||
self.assertEqual(sel.xpath("concat('xpath', 'rules')").getall(),
|
||||
[u'xpathrules'])
|
||||
self.assertEqual([x.extract() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
self.assertEqual([x.get() for x in sel.xpath("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
|
||||
[u'12'])
|
||||
|
||||
def test_root_base_url(self):
|
||||
|
|
@ -60,12 +60,12 @@ class SelectorTestCase(unittest.TestCase):
|
|||
text = b'<div><img src="a.jpg"><p>Hello</div>'
|
||||
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'xml')
|
||||
self.assertEqual(sel.xpath("//div").extract(),
|
||||
self.assertEqual(sel.xpath("//div").getall(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></img></div>'])
|
||||
|
||||
sel = Selector(HtmlResponse('http://example.com', body=text, encoding='utf-8'))
|
||||
self.assertEqual(sel.type, 'html')
|
||||
self.assertEqual(sel.xpath("//div").extract(),
|
||||
self.assertEqual(sel.xpath("//div").getall(),
|
||||
[u'<div><img src="a.jpg"><p>Hello</p></div>'])
|
||||
|
||||
def test_http_header_encoding_precedence(self):
|
||||
|
|
@ -84,15 +84,15 @@ class SelectorTestCase(unittest.TestCase):
|
|||
headers = {'Content-Type': ['text/html; charset=utf-8']}
|
||||
response = HtmlResponse(url="http://example.com", headers=headers, body=html_utf8)
|
||||
x = Selector(response)
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").extract(),
|
||||
self.assertEqual(x.xpath("//span[@id='blank']/text()").getall(),
|
||||
[u'\xa3'])
|
||||
|
||||
def test_badly_encoded_body(self):
|
||||
# \xe9 alone isn't valid utf8 sequence
|
||||
r1 = TextResponse('http://www.example.com', \
|
||||
body=b'<html><p>an Jos\xe9 de</p><html>', \
|
||||
r1 = TextResponse('http://www.example.com',
|
||||
body=b'<html><p>an Jos\xe9 de</p><html>',
|
||||
encoding='utf-8')
|
||||
Selector(r1).xpath('//text()').extract()
|
||||
Selector(r1).xpath('//text()').getall()
|
||||
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
|
|
|
|||
|
|
@ -147,10 +147,10 @@ class XMLFeedSpiderTest(SpiderTest):
|
|||
|
||||
def parse_node(self, response, selector):
|
||||
yield {
|
||||
'loc': selector.xpath('a:loc/text()').extract(),
|
||||
'updated': selector.xpath('b:updated/text()').extract(),
|
||||
'other': selector.xpath('other/@value').extract(),
|
||||
'custom': selector.xpath('other/@b:custom').extract(),
|
||||
'loc': selector.xpath('a:loc/text()').getall(),
|
||||
'updated': selector.xpath('b:updated/text()').getall(),
|
||||
'other': selector.xpath('other/@value').getall(),
|
||||
'custom': selector.xpath('other/@b:custom').getall(),
|
||||
}
|
||||
|
||||
for iterator in ('iternodes', 'xml'):
|
||||
|
|
|
|||
|
|
@ -30,10 +30,13 @@ class XmliterTestCase(unittest.TestCase):
|
|||
response = XmlResponse(url="http://example.com", body=body)
|
||||
attrs = []
|
||||
for x in self.xmliter(response, 'product'):
|
||||
attrs.append((x.xpath("@id").extract(), x.xpath("name/text()").extract(), x.xpath("./type/text()").extract()))
|
||||
attrs.append((
|
||||
x.attrib['id'],
|
||||
x.xpath("name/text()").getall(),
|
||||
x.xpath("./type/text()").getall()))
|
||||
|
||||
self.assertEqual(attrs,
|
||||
[(['001'], ['Name 1'], ['Type 1']), (['002'], ['Name 2'], ['Type 2'])])
|
||||
[('001', ['Name 1'], ['Type 1']), ('002', ['Name 2'], ['Type 2'])])
|
||||
|
||||
def test_xmliter_unusual_node(self):
|
||||
body = b"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
@ -43,7 +46,7 @@ class XmliterTestCase(unittest.TestCase):
|
|||
</root>
|
||||
"""
|
||||
response = XmlResponse(url="http://example.com", body=body)
|
||||
nodenames = [e.xpath('name()').extract()
|
||||
nodenames = [e.xpath('name()').getall()
|
||||
for e in self.xmliter(response, 'matchme...')]
|
||||
self.assertEqual(nodenames, [['matchme...']])
|
||||
|
||||
|
|
@ -93,19 +96,19 @@ class XmliterTestCase(unittest.TestCase):
|
|||
|
||||
attrs = []
|
||||
for x in self.xmliter(r, u'þingflokkur'):
|
||||
attrs.append((x.xpath('@id').extract(),
|
||||
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
|
||||
x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
|
||||
attrs.append((x.attrib['id'],
|
||||
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').getall(),
|
||||
x.xpath(u'./tímabil/fyrstaþing/text()').getall()))
|
||||
|
||||
self.assertEqual(attrs,
|
||||
[([u'26'], [u'-'], [u'80']),
|
||||
([u'21'], [u'Ab'], [u'76']),
|
||||
([u'27'], [u'A'], [u'27'])])
|
||||
[(u'26', [u'-'], [u'80']),
|
||||
(u'21', [u'Ab'], [u'76']),
|
||||
(u'27', [u'A'], [u'27'])])
|
||||
|
||||
def test_xmliter_text(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
|
||||
self.assertEqual([x.xpath("text()").extract() for x in self.xmliter(body, 'product')],
|
||||
self.assertEqual([x.xpath("text()").getall() for x in self.xmliter(body, 'product')],
|
||||
[[u'one'], [u'two']])
|
||||
|
||||
def test_xmliter_namespaces(self):
|
||||
|
|
@ -132,15 +135,15 @@ class XmliterTestCase(unittest.TestCase):
|
|||
|
||||
node = next(my_iter)
|
||||
node.register_namespace('g', 'http://base.google.com/ns/1.0')
|
||||
self.assertEqual(node.xpath('title/text()').extract(), ['Item 1'])
|
||||
self.assertEqual(node.xpath('description/text()').extract(), ['This is item 1'])
|
||||
self.assertEqual(node.xpath('link/text()').extract(), ['http://www.mydummycompany.com/items/1'])
|
||||
self.assertEqual(node.xpath('g:image_link/text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
self.assertEqual(node.xpath('g:id/text()').extract(), ['ITEM_1'])
|
||||
self.assertEqual(node.xpath('g:price/text()').extract(), ['400'])
|
||||
self.assertEqual(node.xpath('image_link/text()').extract(), [])
|
||||
self.assertEqual(node.xpath('id/text()').extract(), [])
|
||||
self.assertEqual(node.xpath('price/text()').extract(), [])
|
||||
self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
|
||||
self.assertEqual(node.xpath('description/text()').getall(), ['This is item 1'])
|
||||
self.assertEqual(node.xpath('link/text()').getall(), ['http://www.mydummycompany.com/items/1'])
|
||||
self.assertEqual(node.xpath('g:image_link/text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
self.assertEqual(node.xpath('g:id/text()').getall(), ['ITEM_1'])
|
||||
self.assertEqual(node.xpath('g:price/text()').getall(), ['400'])
|
||||
self.assertEqual(node.xpath('image_link/text()').getall(), [])
|
||||
self.assertEqual(node.xpath('id/text()').getall(), [])
|
||||
self.assertEqual(node.xpath('price/text()').getall(), [])
|
||||
|
||||
def test_xmliter_exception(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
|
|
@ -159,7 +162,7 @@ class XmliterTestCase(unittest.TestCase):
|
|||
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
|
||||
response = XmlResponse('http://www.example.com', body=body)
|
||||
self.assertEqual(
|
||||
next(self.xmliter(response, 'item')).extract(),
|
||||
next(self.xmliter(response, 'item')).get(),
|
||||
u'<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>'
|
||||
)
|
||||
|
||||
|
|
@ -192,9 +195,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
|
|||
|
||||
namespace_iter = self.xmliter(response, 'image_link', 'http://base.google.com/ns/1.0')
|
||||
node = next(namespace_iter)
|
||||
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
node = next(namespace_iter)
|
||||
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item2.jpg'])
|
||||
self.assertEqual(node.xpath('text()').getall(), ['http://www.mydummycompany.com/images/item2.jpg'])
|
||||
|
||||
def test_xmliter_namespaces_prefix(self):
|
||||
body = b"""\
|
||||
|
|
@ -219,14 +222,14 @@ class LxmlXmliterTestCase(XmliterTestCase):
|
|||
my_iter = self.xmliter(response, 'table', 'http://www.w3.org/TR/html4/', 'h')
|
||||
|
||||
node = next(my_iter)
|
||||
self.assertEqual(len(node.xpath('h:tr/h:td').extract()), 2)
|
||||
self.assertEqual(node.xpath('h:tr/h:td[1]/text()').extract(), ['Apples'])
|
||||
self.assertEqual(node.xpath('h:tr/h:td[2]/text()').extract(), ['Bananas'])
|
||||
self.assertEqual(len(node.xpath('h:tr/h:td').getall()), 2)
|
||||
self.assertEqual(node.xpath('h:tr/h:td[1]/text()').getall(), ['Apples'])
|
||||
self.assertEqual(node.xpath('h:tr/h:td[2]/text()').getall(), ['Bananas'])
|
||||
|
||||
my_iter = self.xmliter(response, 'table', 'http://www.w3schools.com/furniture', 'f')
|
||||
|
||||
node = next(my_iter)
|
||||
self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
|
||||
self.assertEqual(node.xpath('f:name/text()').getall(), ['African Coffee Table'])
|
||||
|
||||
def test_xmliter_objtype_exception(self):
|
||||
i = self.xmliter(42, 'product')
|
||||
|
|
|
|||
3
tox.ini
3
tox.ini
|
|
@ -51,6 +51,9 @@ deps =
|
|||
cssselect==0.9.1
|
||||
zope.interface==4.1.1
|
||||
-rtests/requirements-py2.txt
|
||||
# Not used directly but allows boto GCE plugins to load.
|
||||
# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
|
||||
google-compute-engine==2.8.12
|
||||
|
||||
[testenv:trunk]
|
||||
basepython = python2.7
|
||||
|
|
|
|||
Loading…
Reference in New Issue