mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master'
This commit is contained in:
Taito Horiuchi
commit
af23c920bf
|
|
@ -1,29 +1,7 @@
|
|||
[bumpversion]
|
||||
current_version = 1.3.0
|
||||
current_version = 1.3.2
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
parse = ^
|
||||
(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)
|
||||
(?:(?P<prerel>[abc]|rc|dev)(?P<prerelversion>\d+))?
|
||||
serialize =
|
||||
{major}.{minor}.{patch}{prerel}{prerelversion}
|
||||
{major}.{minor}.{patch}
|
||||
|
||||
[bumpversion:file:scrapy/VERSION]
|
||||
|
||||
[bumpversion:part:prerel]
|
||||
optional_value = gamma
|
||||
values =
|
||||
dev
|
||||
rc
|
||||
gamma
|
||||
|
||||
[bumpversion:part:prerelversion]
|
||||
values =
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ dist
|
|||
.idea
|
||||
htmlcov/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache/
|
||||
|
||||
# Windows
|
||||
Thumbs.db
|
||||
|
|
|
|||
47
.travis.yml
47
.travis.yml
|
|
@ -1,19 +1,46 @@
|
|||
language: python
|
||||
python: 3.5
|
||||
sudo: false
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /^\d\.\d+$/
|
||||
- /^\d\.\d+\.\d+(rc\d+|dev\d+)?$/
|
||||
env:
|
||||
- TOXENV=py27
|
||||
- TOXENV=jessie
|
||||
- TOXENV=py33
|
||||
- TOXENV=py35
|
||||
- TOXENV=docs
|
||||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- python: 2.7
|
||||
env: TOXENV=py27
|
||||
- python: 2.7
|
||||
env: TOXENV=jessie
|
||||
- python: 3.3
|
||||
env: TOXENV=py33
|
||||
- python: 3.5
|
||||
env: TOXENV=py35
|
||||
- python: 3.6
|
||||
env: TOXENV=py36
|
||||
- python: 2.7
|
||||
env: TOXENV=pypy
|
||||
- python: 3.6
|
||||
env: TOXENV=docs
|
||||
allow_failures:
|
||||
- python: 2.7
|
||||
env: TOXENV=pypy
|
||||
install:
|
||||
- pip install -U tox twine wheel codecov
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy" ]; then
|
||||
export PYENV_ROOT="$HOME/.pyenv"
|
||||
if [ -f "$PYENV_ROOT/bin/pyenv" ]; then
|
||||
pushd "$PYENV_ROOT" && git pull && popd
|
||||
else
|
||||
rm -rf "$PYENV_ROOT" && git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
|
||||
fi
|
||||
# get latest PyPy from pyenv directly (thanks to natural version sort option -V)
|
||||
export PYPY_VERSION=`"$PYENV_ROOT/bin/pyenv" install --list |grep -o -E 'pypy-[0-9][\.0-9]*$' |sort -V |tail -1`
|
||||
"$PYENV_ROOT/bin/pyenv" install --skip-existing "$PYPY_VERSION"
|
||||
virtualenv --python="$PYENV_ROOT/versions/$PYPY_VERSION/bin/python" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
- pip install -U tox twine wheel codecov
|
||||
|
||||
script: tox
|
||||
after_success:
|
||||
- codecov
|
||||
|
|
@ -35,4 +62,4 @@ deploy:
|
|||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|dev[0-9]+)?$"
|
||||
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
|
|
|
|||
|
|
@ -13,10 +13,6 @@ Scrapy
|
|||
.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg
|
||||
:target: https://pypi.python.org/pypi/Scrapy
|
||||
:alt: Wheel Status
|
||||
|
||||
.. image:: http://static.scrapy.org/py3progress/badge.svg
|
||||
:target: https://github.com/scrapy/scrapy/wiki/Python-3-Porting
|
||||
:alt: Python 3 Porting Status
|
||||
|
||||
.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
|
||||
:target: http://codecov.io/github/scrapy/scrapy?branch=master
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
:orphan:
|
||||
|
||||
Scrapy artwork
|
||||
==============
|
||||
|
||||
|
|
@ -1,3 +1,5 @@
|
|||
:orphan:
|
||||
|
||||
======================================
|
||||
Scrapy documentation quick start guide
|
||||
======================================
|
||||
|
|
@ -40,8 +40,7 @@ http://quotes.toscrape.com, following the pagination::
|
|||
|
||||
next_page = response.css('li.next a::attr("href")').extract_first()
|
||||
if next_page is not None:
|
||||
next_page = response.urljoin(next_page)
|
||||
yield scrapy.Request(next_page, callback=self.parse)
|
||||
yield response.follow(next_page, self.parse)
|
||||
|
||||
|
||||
Put this in a text file, name it to something like ``quotes_spider.py``
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ You will see something like::
|
|||
[s] shelp() Shell help (print this help)
|
||||
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
|
||||
[s] view(response) View response in a browser
|
||||
>>>
|
||||
>>>
|
||||
|
||||
Using the shell, you can try selecting elements using `CSS`_ with the response
|
||||
object::
|
||||
|
|
@ -399,7 +399,7 @@ quotes elements and put them together into a Python dictionary::
|
|||
>>>
|
||||
|
||||
Extracting data in our spider
|
||||
------------------------------
|
||||
-----------------------------
|
||||
|
||||
Let's get back to our spider. Until now, it doesn't extract any data in
|
||||
particular, just saves the whole HTML page to a local file. Let's integrate the
|
||||
|
|
@ -423,7 +423,7 @@ in the callback, as you can see below::
|
|||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('span small::text').extract_first(),
|
||||
'author': quote.css('small.author::text').extract_first(),
|
||||
'tags': quote.css('div.tags a.tag::text').extract(),
|
||||
}
|
||||
|
||||
|
|
@ -522,7 +522,7 @@ page, extracting data from it::
|
|||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('span small::text').extract_first(),
|
||||
'author': quote.css('small.author::text').extract_first(),
|
||||
'tags': quote.css('div.tags a.tag::text').extract(),
|
||||
}
|
||||
|
||||
|
|
@ -551,13 +551,65 @@ In our example, it creates a sort of loop, following all the links to the next p
|
|||
until it doesn't find one -- handy for crawling blogs, forums and other sites with
|
||||
pagination.
|
||||
|
||||
|
||||
.. _response-follow-example:
|
||||
|
||||
A shortcut for creating Requests
|
||||
--------------------------------
|
||||
|
||||
As a shortcut for creating Request objects you can use
|
||||
:meth:`response.follow <scrapy.http.TextResponse.follow>`::
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
class QuotesSpider(scrapy.Spider):
|
||||
name = "quotes"
|
||||
start_urls = [
|
||||
'http://quotes.toscrape.com/page/1/',
|
||||
]
|
||||
|
||||
def parse(self, response):
|
||||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('span small::text').extract_first(),
|
||||
'tags': quote.css('div.tags a.tag::text').extract(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr(href)').extract_first()
|
||||
if next_page is not None:
|
||||
yield response.follow(next_page, callback=self.parse)
|
||||
|
||||
Unlike scrapy.Request, ``response.follow`` supports relative URLs directly - no
|
||||
need to call urljoin. Note that ``response.follow`` just returns a Request
|
||||
instance; you still have to yield this Request.
|
||||
|
||||
You can also pass a selector to ``response.follow`` instead of a string;
|
||||
this selector should extract necessary attributes::
|
||||
|
||||
for href in response.css('li.next a::attr(href)'):
|
||||
yield response.follow(href, callback=self.parse)
|
||||
|
||||
For ``<a>`` elements there is a shortcut: ``response.follow`` uses their href
|
||||
attribute automatically. So the code can be shortened further::
|
||||
|
||||
for a in response.css('li.next a'):
|
||||
yield response.follow(a, callback=self.parse)
|
||||
|
||||
.. note::
|
||||
|
||||
``response.follow(response.css('li.next a'))`` is not valid because
|
||||
``response.css`` returns a list-like object with selectors for all results,
|
||||
not a single selector. A ``for`` loop like in the example above, or
|
||||
``response.follow(response.css('li.next a')[0])`` is fine.
|
||||
|
||||
More examples and patterns
|
||||
--------------------------
|
||||
|
||||
Here is another spider that illustrates callbacks and following links,
|
||||
this time for scraping author information::
|
||||
|
||||
|
||||
import scrapy
|
||||
|
||||
|
||||
|
|
@ -568,15 +620,12 @@ this time for scraping author information::
|
|||
|
||||
def parse(self, response):
|
||||
# follow links to author pages
|
||||
for href in response.css('.author+a::attr(href)').extract():
|
||||
yield scrapy.Request(response.urljoin(href),
|
||||
callback=self.parse_author)
|
||||
for href in response.css('.author + a::attr(href)'):
|
||||
yield response.follow(href, self.parse_author)
|
||||
|
||||
# follow pagination links
|
||||
next_page = response.css('li.next a::attr(href)').extract_first()
|
||||
if next_page is not None:
|
||||
next_page = response.urljoin(next_page)
|
||||
yield scrapy.Request(next_page, callback=self.parse)
|
||||
for href in response.css('li.next a::attr(href)'):
|
||||
yield response.follow(href, self.parse)
|
||||
|
||||
def parse_author(self, response):
|
||||
def extract_with_css(query):
|
||||
|
|
@ -592,6 +641,9 @@ This spider will start from the main page, it will follow all the links to the
|
|||
authors pages calling the ``parse_author`` callback for each of them, and also
|
||||
the pagination links with the ``parse`` callback as we saw before.
|
||||
|
||||
Here we're passing callbacks to ``response.follow`` as positional arguments
|
||||
to make the code shorter; it also works for ``scrapy.Request``.
|
||||
|
||||
The ``parse_author`` callback defines a helper function to extract and cleanup the
|
||||
data from a CSS query and yields the Python dict with the author data.
|
||||
|
||||
|
|
@ -624,7 +676,7 @@ option when running them::
|
|||
scrapy crawl quotes -o quotes-humor.json -a tag=humor
|
||||
|
||||
These arguments are passed to the Spider's ``__init__`` method and become
|
||||
spider attributes by default.
|
||||
spider attributes by default.
|
||||
|
||||
In this example, the value provided for the ``tag`` argument will be available
|
||||
via ``self.tag``. You can use this to make your spider fetch only quotes
|
||||
|
|
@ -647,13 +699,12 @@ with a specific tag, building the URL based on the argument::
|
|||
for quote in response.css('div.quote'):
|
||||
yield {
|
||||
'text': quote.css('span.text::text').extract_first(),
|
||||
'author': quote.css('span small a::text').extract_first(),
|
||||
'author': quote.css('small.author::text').extract_first(),
|
||||
}
|
||||
|
||||
next_page = response.css('li.next a::attr(href)').extract_first()
|
||||
if next_page is not None:
|
||||
next_page = response.urljoin(next_page)
|
||||
yield scrapy.Request(next_page, self.parse)
|
||||
yield response.follow(next_page, self.parse)
|
||||
|
||||
|
||||
If you pass the ``tag=humor`` argument to this spider, you'll notice that it
|
||||
|
|
|
|||
|
|
@ -3,6 +3,64 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
Scrapy 1.3.2 (2017-02-13)
|
||||
-------------------------
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Preserve crequest class when converting to/from dicts (utils.reqser) (:issue:`2510`).
|
||||
- Use consistent selectors for author field in tutorial (:issue:`2551`).
|
||||
- Fix TLS compatibility in Twisted 17+ (:issue:`2558`)
|
||||
|
||||
Scrapy 1.3.1 (2017-02-08)
|
||||
-------------------------
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
- Support ``'True'`` and ``'False'`` string values for boolean settings (:issue:`2519`);
|
||||
you can now do something like ``scrapy crawl myspider -s REDIRECT_ENABLED=False``.
|
||||
- Support kwargs with ``response.xpath()`` to use :ref:`XPath variables <topics-selectors-xpath-variables>`
|
||||
and ad-hoc namespaces declarations ;
|
||||
this requires at least Parsel v1.1 (:issue:`2457`).
|
||||
- Add support for Python 3.6 (:issue:`2485`).
|
||||
- Run tests on PyPy (warning: some tests still fail, so PyPy is not supported yet).
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
- Enforce ``DNS_TIMEOUT`` setting (:issue:`2496`).
|
||||
- Fix :command:`view` command ; it was a regression in v1.3.0 (:issue:`2503`).
|
||||
- Fix tests regarding ``*_EXPIRES settings`` with Files/Images pipelines (:issue:`2460`).
|
||||
- Fix name of generated pipeline class when using basic project template (:issue:`2466`).
|
||||
- Fix compatiblity with Twisted 17+ (:issue:`2496`, :issue:`2528`).
|
||||
- Fix ``scrapy.Item`` inheritance on Python 3.6 (:issue:`2511`).
|
||||
- Enforce numeric values for components order in ``SPIDER_MIDDLEWARES``,
|
||||
``DOWNLOADER_MIDDLEWARES``, ``EXTENIONS`` and ``SPIDER_CONTRACTS`` (:issue:`2420`).
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
- Reword Code of Coduct section and upgrade to Contributor Covenant v1.4
|
||||
(:issue:`2469`).
|
||||
- Clarify that passing spider arguments converts them to spider attributes
|
||||
(:issue:`2483`).
|
||||
- Document ``formid`` argument on ``FormRequest.from_response()`` (:issue:`2497`).
|
||||
- Add .rst extension to README files (:issue:`2507`).
|
||||
- Mention LevelDB cache storage backend (:issue:`2525`).
|
||||
- Use ``yield`` in sample callback code (:issue:`2533`).
|
||||
- Add note about HTML entities decoding with ``.re()/.re_first()`` (:issue:`1704`).
|
||||
- Typos (:issue:`2512`, :issue:`2534`, :issue:`2531`).
|
||||
|
||||
Cleanups
|
||||
~~~~~~~~
|
||||
|
||||
- Remove reduntant check in ``MetaRefreshMiddleware`` (:issue:`2542`).
|
||||
- Faster checks in ``LinkExtractor`` for allow/deny patterns (:issue:`2538`).
|
||||
- Remove dead code supporting old Twisted versions (:issue:`2544`).
|
||||
|
||||
|
||||
Scrapy 1.3.0 (2016-12-21)
|
||||
-------------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -358,6 +358,12 @@ Opens the given URL in a browser, as your Scrapy spider would "see" it.
|
|||
Sometimes spiders see pages differently from regular users, so this can be used
|
||||
to check what the spider "sees" and confirm it's what you expect.
|
||||
|
||||
Supported options:
|
||||
|
||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||
|
||||
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
|
||||
|
||||
Usage example::
|
||||
|
||||
$ scrapy view http://www.example.com/some/page.html
|
||||
|
|
|
|||
|
|
@ -318,10 +318,11 @@ HttpCacheMiddleware
|
|||
This middleware provides low-level cache to all HTTP requests and responses.
|
||||
It has to be combined with a cache storage backend as well as a cache policy.
|
||||
|
||||
Scrapy ships with two HTTP cache storage backends:
|
||||
Scrapy ships with three HTTP cache storage backends:
|
||||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
* :ref:`httpcache-storage-leveldb`
|
||||
|
||||
You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
|
||||
setting. Or you can also implement your own storage backend.
|
||||
|
|
@ -680,7 +681,9 @@ HttpProxyMiddleware
|
|||
* ``no_proxy``
|
||||
|
||||
You can also set the meta key ``proxy`` per-request, to a value like
|
||||
``http://some_proxy_server:port``.
|
||||
``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``.
|
||||
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
|
||||
environment variables, and it will also ignore ``no_proxy`` environment variable.
|
||||
|
||||
.. _urllib: https://docs.python.org/2/library/urllib.html
|
||||
.. _urllib2: https://docs.python.org/2/library/urllib2.html
|
||||
|
|
@ -748,7 +751,7 @@ REDIRECT_MAX_TIMES
|
|||
|
||||
Default: ``20``
|
||||
|
||||
The maximum number of redirections that will be follow for a single request.
|
||||
The maximum number of redirections that will be followed for a single request.
|
||||
|
||||
MetaRefreshMiddleware
|
||||
---------------------
|
||||
|
|
@ -948,8 +951,16 @@ enable it for :ref:`broad crawls <topics-broad-crawls>`.
|
|||
HttpProxyMiddleware settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. setting:: HTTPPROXY_ENABLED
|
||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||
|
||||
HTTPPROXY_ENABLED
|
||||
^^^^^^^^^^^^^^^^^
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether or not to enable the :class:`HttpProxyMiddleware`.
|
||||
|
||||
HTTPPROXY_AUTH_ENCODING
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
|
|
|
|||
|
|
@ -225,7 +225,8 @@ XmlItemExporter
|
|||
|
||||
Exports Items in XML format to the specified file object.
|
||||
|
||||
:param file: the file-like object to use for exporting the data.
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
:param root_element: The name of root element in the exported XML.
|
||||
:type root_element: str
|
||||
|
|
@ -281,7 +282,8 @@ CsvItemExporter
|
|||
CSV columns and their order. The :attr:`export_empty_fields` attribute has
|
||||
no effect on this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data.
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
:param include_headers_line: If enabled, makes the exporter output a header
|
||||
line with the field names taken from
|
||||
|
|
@ -312,7 +314,8 @@ PickleItemExporter
|
|||
|
||||
Exports Items in pickle format to the given file-like object.
|
||||
|
||||
:param file: the file-like object to use for exporting the data.
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
:param protocol: The pickle protocol to use.
|
||||
:type protocol: int
|
||||
|
|
@ -333,7 +336,8 @@ PprintItemExporter
|
|||
|
||||
Exports Items in pretty print format to the specified file object.
|
||||
|
||||
:param file: the file-like object to use for exporting the data.
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
The additional keyword arguments of this constructor are passed to the
|
||||
:class:`BaseItemExporter` constructor.
|
||||
|
|
@ -356,7 +360,8 @@ JsonItemExporter
|
|||
arguments to the `JSONEncoder`_ constructor, so you can use any
|
||||
`JSONEncoder`_ constructor argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data.
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
@ -386,7 +391,8 @@ JsonLinesItemExporter
|
|||
the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
|
||||
constructor argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data.
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ LxmlLinkExtractor
|
|||
:synopsis: lxml's HTMLParser-based link extractors
|
||||
|
||||
|
||||
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None)
|
||||
.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, strip=True)
|
||||
|
||||
LxmlLinkExtractor is the recommended link extractor with handy filtering
|
||||
options. It is implemented using lxml's robust HTMLParser.
|
||||
|
|
@ -132,4 +132,13 @@ LxmlLinkExtractor
|
|||
|
||||
:type process_value: callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
must be stripped from ``href`` attributes of ``<a>``, ``<area>``
|
||||
and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
|
||||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: boolean
|
||||
|
||||
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
|
||||
|
|
|
|||
|
|
@ -121,6 +121,9 @@ Request objects
|
|||
see :ref:`topics-request-response-ref-errbacks` below.
|
||||
:type errback: callable
|
||||
|
||||
:param flags: Flags sent to the request, can be used for logging or similar purposes.
|
||||
:type flags: list
|
||||
|
||||
.. attribute:: Request.url
|
||||
|
||||
A string containing the URL of this request. Keep in mind that this
|
||||
|
|
@ -207,12 +210,12 @@ different fields from different pages::
|
|||
request = scrapy.Request("http://www.example.com/some_page.html",
|
||||
callback=self.parse_page2)
|
||||
request.meta['item'] = item
|
||||
return request
|
||||
yield request
|
||||
|
||||
def parse_page2(self, response):
|
||||
item = response.meta['item']
|
||||
item['other_url'] = response.url
|
||||
return item
|
||||
yield item
|
||||
|
||||
|
||||
.. _topics-request-response-ref-errbacks:
|
||||
|
|
@ -358,7 +361,7 @@ fields with form data from :class:`Response` objects.
|
|||
The :class:`FormRequest` objects support the following class method in
|
||||
addition to the standard :class:`Request` methods:
|
||||
|
||||
.. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
|
||||
.. classmethod:: FormRequest.from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
|
||||
|
||||
Returns a new :class:`FormRequest` object with its form field values
|
||||
pre-populated with those found in the HTML ``<form>`` element contained
|
||||
|
|
@ -376,6 +379,10 @@ fields with form data from :class:`Response` objects.
|
|||
control clicked (instead of disabling it) you can also use the
|
||||
``clickdata`` argument.
|
||||
|
||||
.. caution:: Using this method with select elements which have leading
|
||||
or trailing whitespace in the option values will not work due to a
|
||||
`bug in lxml`_, which should be fixed in lxml 3.8 and above.
|
||||
|
||||
:param response: the response containing a HTML form which will be used
|
||||
to pre-populate the form fields
|
||||
:type response: :class:`Response` object
|
||||
|
|
@ -383,6 +390,9 @@ fields with form data from :class:`Response` objects.
|
|||
:param formname: if given, the form with name attribute set to this value will be used.
|
||||
:type formname: string
|
||||
|
||||
:param formid: if given, the form with id attribute set to this value will be used.
|
||||
:type formid: string
|
||||
|
||||
:param formxpath: if given, the first form that matches the xpath will be used.
|
||||
:type formxpath: string
|
||||
|
||||
|
|
@ -421,6 +431,9 @@ fields with form data from :class:`Response` objects.
|
|||
.. versionadded:: 1.1.0
|
||||
The ``formcss`` parameter.
|
||||
|
||||
.. versionadded:: 1.1.0
|
||||
The ``formid`` parameter.
|
||||
|
||||
Request usage examples
|
||||
----------------------
|
||||
|
||||
|
|
@ -591,6 +604,9 @@ Response objects
|
|||
|
||||
urlparse.urljoin(response.url, url)
|
||||
|
||||
.. automethod:: Response.follow
|
||||
|
||||
|
||||
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
|
||||
|
||||
.. _topics-request-response-ref-response-subclasses:
|
||||
|
|
@ -677,6 +693,8 @@ TextResponse objects
|
|||
|
||||
response.css('p')
|
||||
|
||||
.. automethod:: TextResponse.follow
|
||||
|
||||
.. method:: TextResponse.body_as_unicode()
|
||||
|
||||
The same as :attr:`text`, but available as a method. This method is
|
||||
|
|
@ -704,3 +722,4 @@ XmlResponse objects
|
|||
line. See :attr:`TextResponse.encoding`.
|
||||
|
||||
.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
|
||||
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
|
||||
|
|
|
|||
|
|
@ -283,6 +283,40 @@ XPath specification.
|
|||
|
||||
.. _Location Paths: https://www.w3.org/TR/xpath#location-paths
|
||||
|
||||
.. _topics-selectors-xpath-variables:
|
||||
|
||||
Variables in XPath expressions
|
||||
------------------------------
|
||||
|
||||
XPath allows you to reference variables in your XPath expressions, using
|
||||
the ``$somevariable`` syntax. This is somewhat similar to parameterized
|
||||
queries or prepared statements in the SQL world where you replace
|
||||
some arguments in your queries with placeholders like ``?``,
|
||||
which are then substituted with values passed with the query.
|
||||
|
||||
Here's an example to match an element based on its "id" attribute value,
|
||||
without hard-coding it (that was shown previously)::
|
||||
|
||||
>>> # `$val` used in the expression, a `val` argument needs to be passed
|
||||
>>> response.xpath('//div[@id=$val]/a/text()', val='images').extract_first()
|
||||
u'Name: My image 1 '
|
||||
|
||||
Here's another example, to find the "id" attribute of a ``<div>`` tag containing
|
||||
five ``<a>`` children (here we pass the value ``5`` as an integer)::
|
||||
|
||||
>>> response.xpath('//div[count(a)=$cnt]/@id', cnt=5).extract_first()
|
||||
u'images'
|
||||
|
||||
All variable references must have a binding value when calling ``.xpath()``
|
||||
(otherwise you'll get a ``ValueError: XPath error:`` exception).
|
||||
This is done by passing as many named arguments as necessary.
|
||||
|
||||
`parsel`_, the library powering Scrapy selectors, has more details and examples
|
||||
on `XPath variables`_.
|
||||
|
||||
.. _parsel: https://parsel.readthedocs.io/
|
||||
.. _XPath variables: https://parsel.readthedocs.io/en/latest/usage.html#variables-in-xpath-expressions
|
||||
|
||||
Using EXSLT extensions
|
||||
----------------------
|
||||
|
||||
|
|
@ -626,6 +660,10 @@ Built-in Selectors reference
|
|||
``regex`` can be either a compiled regular expression or a string which
|
||||
will be compiled to a regular expression using ``re.compile(regex)``
|
||||
|
||||
.. note::
|
||||
|
||||
Note that ``re()`` and ``re_first()`` both decode HTML entities (except ``<`` and ``&``).
|
||||
|
||||
.. method:: register_namespace(prefix, uri)
|
||||
|
||||
Register the given namespace to be used in this :class:`Selector`.
|
||||
|
|
|
|||
|
|
@ -686,6 +686,42 @@ The Feed Temp dir allows you to set a custom folder to save crawler
|
|||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: FTP_PASSIVE_MODE
|
||||
|
||||
FTP_PASSIVE_MODE
|
||||
----------------
|
||||
|
||||
Default: ``True``
|
||||
|
||||
Whether or not to use passive mode when initiating FTP transfers.
|
||||
|
||||
.. setting:: FTP_PASSWORD
|
||||
|
||||
FTP_PASSWORD
|
||||
------------
|
||||
|
||||
Default: ``"guest"``
|
||||
|
||||
The password to use for FTP connections when there is no ``"ftp_password"``
|
||||
in ``Request`` meta.
|
||||
|
||||
.. note::
|
||||
Paraphrasing `RFC 1635`_, although it is common to use either the password
|
||||
"guest" or one's e-mail address for anonymous FTP,
|
||||
some FTP servers explicitly ask for the user's e-mail address
|
||||
and will not allow login with the "guest" password.
|
||||
|
||||
.. _RFC 1635: https://tools.ietf.org/html/rfc1635
|
||||
|
||||
.. setting:: FTP_USER
|
||||
|
||||
FTP_USER
|
||||
--------
|
||||
|
||||
Default: ``"anonymous"``
|
||||
|
||||
The username to use for FTP connections when there is no ``"ftp_user"``
|
||||
in ``Request`` meta.
|
||||
|
||||
.. setting:: ITEM_PIPELINES
|
||||
|
||||
|
|
@ -827,13 +863,15 @@ Example::
|
|||
MEMUSAGE_ENABLED
|
||||
----------------
|
||||
|
||||
Default: ``False``
|
||||
Default: ``True``
|
||||
|
||||
Scope: ``scrapy.extensions.memusage``
|
||||
|
||||
Whether to enable the memory usage extension that will shutdown the Scrapy
|
||||
process when it exceeds a memory limit, and also notify by email when that
|
||||
happened.
|
||||
Whether to enable the memory usage extension. This extension keeps track of
|
||||
a peak memory used by the process (it writes it to stats). It can also
|
||||
optionally shutdown the Scrapy process when it exceeds a memory limit
|
||||
(see :setting:`MEMUSAGE_LIMIT_MB`), and notify by email when that happened
|
||||
(see :setting:`MEMUSAGE_NOTIFY_MAIL`).
|
||||
|
||||
See :ref:`topics-extensions-ref-memusage`.
|
||||
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ following methods:
|
|||
|
||||
.. method:: process_spider_exception(response, exception, spider)
|
||||
|
||||
This method is called when when a spider or :meth:`process_spider_input`
|
||||
This method is called when a spider or :meth:`process_spider_input`
|
||||
method (from other spider middleware) raises an exception.
|
||||
|
||||
:meth:`process_spider_exception` should return either ``None`` or an
|
||||
|
|
|
|||
|
|
@ -144,16 +144,12 @@ scrapy.Spider
|
|||
.. method:: start_requests()
|
||||
|
||||
This method must return an iterable with the first Requests to crawl for
|
||||
this spider.
|
||||
this spider. It is called by Scrapy when the spider is opened for
|
||||
scraping. Scrapy calls it only once, so it is safe to implement
|
||||
:meth:`start_requests` as a generator.
|
||||
|
||||
This is the method called by Scrapy when the spider is opened for
|
||||
scraping when no particular URLs are specified. If particular URLs are
|
||||
specified, the :meth:`make_requests_from_url` is used instead to create
|
||||
the Requests. This method is also called only once from Scrapy, so it's
|
||||
safe to implement it as a generator.
|
||||
|
||||
The default implementation uses :meth:`make_requests_from_url` to
|
||||
generate Requests for each url in :attr:`start_urls`.
|
||||
The default implementation generates ``Request(url, dont_filter=True)``
|
||||
for each url in :attr:`start_urls`.
|
||||
|
||||
If you want to change the Requests used to start scraping a domain, this is
|
||||
the method to override. For example, if you need to start by logging in using
|
||||
|
|
@ -172,18 +168,6 @@ scrapy.Spider
|
|||
# each of them, with another callback
|
||||
pass
|
||||
|
||||
.. method:: make_requests_from_url(url)
|
||||
|
||||
A method that receives a URL and returns a :class:`~scrapy.http.Request`
|
||||
object (or a list of :class:`~scrapy.http.Request` objects) to scrape. This
|
||||
method is used to construct the initial requests in the
|
||||
:meth:`start_requests` method, and is typically used to convert urls to
|
||||
requests.
|
||||
|
||||
Unless overridden, this method returns Requests with the :meth:`parse`
|
||||
method as their callback function, and with dont_filter parameter enabled
|
||||
(see :class:`~scrapy.http.Request` class for more info).
|
||||
|
||||
.. method:: parse(response)
|
||||
|
||||
This is the default callback used by Scrapy to process downloaded
|
||||
|
|
|
|||
|
|
@ -28,16 +28,16 @@ Query Scrapy settings
|
|||
Print raw setting value
|
||||
.TP
|
||||
.I --getbool=SETTING
|
||||
Print setting value, intepreted as a boolean
|
||||
Print setting value, interpreted as a boolean
|
||||
.TP
|
||||
.I --getint=SETTING
|
||||
Print setting value, intepreted as an integer
|
||||
Print setting value, interpreted as an integer
|
||||
.TP
|
||||
.I --getfloat=SETTING
|
||||
Print setting value, intepreted as an float
|
||||
Print setting value, interpreted as a float
|
||||
.TP
|
||||
.I --getlist=SETTING
|
||||
Print setting value, intepreted as an float
|
||||
Print setting value, interpreted as a float
|
||||
.TP
|
||||
.I --init
|
||||
Print initial setting value (before loading extensions and spiders)
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@ lxml>=3.2.4
|
|||
pyOpenSSL>=0.13.1
|
||||
cssselect>=0.9
|
||||
queuelib>=1.1.1
|
||||
w3lib>=1.14.2
|
||||
w3lib>=1.17.0
|
||||
service_identity
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@ Twisted>=13.1.0
|
|||
lxml
|
||||
pyOpenSSL
|
||||
cssselect>=0.9
|
||||
w3lib>=1.15.0
|
||||
w3lib>=1.17.0
|
||||
queuelib
|
||||
six>=1.5.2
|
||||
PyDispatcher>=2.0.5
|
||||
service_identity
|
||||
parsel>=0.9.5
|
||||
parsel>=1.1
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
1.3.0
|
||||
1.3.2
|
||||
|
|
|
|||
|
|
@ -11,9 +11,8 @@ class Command(fetch.Command):
|
|||
"contents in a browser"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider",
|
||||
help="use this spider")
|
||||
super(Command, self).add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
open_in_browser(response)
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
from OpenSSL import SSL
|
||||
from twisted.internet.ssl import ClientContextFactory
|
||||
|
||||
try:
|
||||
from scrapy import twisted_version
|
||||
|
||||
if twisted_version >= (14, 0, 0):
|
||||
|
||||
from zope.interface.declarations import implementer
|
||||
|
||||
# the following should be available from Twisted 14.0.0
|
||||
from twisted.internet.ssl import (optionsForClientTLS,
|
||||
CertificateOptions,
|
||||
platformTrust)
|
||||
|
||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
|
||||
|
|
@ -86,7 +86,7 @@ try:
|
|||
'method': self._ssl_method,
|
||||
})
|
||||
|
||||
except ImportError:
|
||||
else:
|
||||
|
||||
class ScrapyClientContextFactory(ClientContextFactory):
|
||||
"A SSL context factory which is more permissive against SSL bugs."
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
|
|||
|
||||
import re
|
||||
from io import BytesIO
|
||||
from six.moves.urllib.parse import urlparse, unquote
|
||||
from six.moves.urllib.parse import unquote
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.protocols.ftp import FTPClient, CommandFailed
|
||||
|
|
@ -38,6 +38,8 @@ from twisted.internet.protocol import Protocol, ClientCreator
|
|||
|
||||
from scrapy.http import Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
||||
|
||||
class ReceivedDataProtocol(Protocol):
|
||||
def __init__(self, filename=None):
|
||||
|
|
@ -64,14 +66,19 @@ class FTPDownloadHandler(object):
|
|||
"default": 503,
|
||||
}
|
||||
|
||||
def __init__(self, setting):
|
||||
pass
|
||||
def __init__(self, settings):
|
||||
self.default_user = settings['FTP_USER']
|
||||
self.default_password = settings['FTP_PASSWORD']
|
||||
self.passive_mode = settings['FTP_PASSIVE_MODE']
|
||||
|
||||
def download_request(self, request, spider):
|
||||
parsed_url = urlparse(request.url)
|
||||
creator = ClientCreator(reactor, FTPClient, request.meta["ftp_user"],
|
||||
request.meta["ftp_password"],
|
||||
passive=request.meta.get("ftp_passive", 1))
|
||||
parsed_url = urlparse_cached(request)
|
||||
user = request.meta.get("ftp_user", self.default_user)
|
||||
password = request.meta.get("ftp_password", self.default_password)
|
||||
passive_mode = 1 if bool(request.meta.get("ftp_passive",
|
||||
self.passive_mode)) else 0
|
||||
creator = ClientCreator(reactor, FTPClient, user, password,
|
||||
passive=passive_mode)
|
||||
return creator.connectTCP(parsed_url.hostname, parsed_url.port or 21).addCallback(self.gotClient,
|
||||
request, unquote(parsed_url.path))
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,6 @@
|
|||
from scrapy import twisted_version
|
||||
from __future__ import absolute_import
|
||||
from .http10 import HTTP10DownloadHandler
|
||||
|
||||
if twisted_version >= (11, 1, 0):
|
||||
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
|
||||
else:
|
||||
HTTPDownloadHandler = HTTP10DownloadHandler
|
||||
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
|
||||
|
||||
|
||||
# backwards compatibility
|
||||
|
|
|
|||
|
|
@ -105,6 +105,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
self._tunneledHost = host
|
||||
self._tunneledPort = port
|
||||
self._contextFactory = contextFactory
|
||||
self._connectBuffer = bytearray()
|
||||
|
||||
def requestTunnel(self, protocol):
|
||||
"""Asks the proxy to open a tunnel."""
|
||||
|
|
@ -121,8 +122,16 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
created, notifies the client that we are ready to send requests. If not
|
||||
raises a TunnelError.
|
||||
"""
|
||||
self._connectBuffer += rcvd_bytes
|
||||
# make sure that enough (all) bytes are consumed
|
||||
# and that we've got all HTTP headers (ending with a blank line)
|
||||
# from the proxy so that we don't send those bytes to the TLS layer
|
||||
#
|
||||
# see https://github.com/scrapy/scrapy/issues/2491
|
||||
if b'\r\n\r\n' not in self._connectBuffer:
|
||||
return
|
||||
self._protocol.dataReceived = self._protocolDataReceived
|
||||
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)
|
||||
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
|
||||
if respm and int(respm.group('status')) == 200:
|
||||
try:
|
||||
# this sets proper Server Name Indication extension
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import logging
|
||||
from OpenSSL import SSL
|
||||
|
||||
from scrapy import twisted_version
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -18,11 +20,17 @@ openssl_methods = {
|
|||
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
|
||||
}
|
||||
|
||||
# ClientTLSOptions requires a recent-enough version of Twisted
|
||||
try:
|
||||
if twisted_version >= (14, 0, 0):
|
||||
# ClientTLSOptions requires a recent-enough version of Twisted.
|
||||
# Not having ScrapyClientTLSOptions should not matter for older
|
||||
# Twisted versions because it is not used in the fallback
|
||||
# ScrapyClientContextFactory.
|
||||
|
||||
# taken from twisted/twisted/internet/_sslverify.py
|
||||
|
||||
try:
|
||||
# XXX: this try-except is not needed in Twisted 17.0.0+ because
|
||||
# it requires pyOpenSSL 0.16+.
|
||||
from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
|
||||
except ImportError:
|
||||
SSL_CB_HANDSHAKE_START = 0x10
|
||||
|
|
@ -30,10 +38,17 @@ try:
|
|||
|
||||
from twisted.internet.ssl import AcceptableCiphers
|
||||
from twisted.internet._sslverify import (ClientTLSOptions,
|
||||
_maybeSetHostNameIndication,
|
||||
verifyHostname,
|
||||
VerificationError)
|
||||
|
||||
if twisted_version < (17, 0, 0):
|
||||
from twisted.internet._sslverify import _maybeSetHostNameIndication
|
||||
set_tlsext_host_name = _maybeSetHostNameIndication
|
||||
else:
|
||||
def set_tlsext_host_name(connection, hostNameBytes):
|
||||
connection.set_tlsext_host_name(hostNameBytes)
|
||||
|
||||
|
||||
class ScrapyClientTLSOptions(ClientTLSOptions):
|
||||
"""
|
||||
SSL Client connection creator ignoring certificate verification errors
|
||||
|
|
@ -46,7 +61,7 @@ try:
|
|||
|
||||
def _identityVerifyingInfoCallback(self, connection, where, ret):
|
||||
if where & SSL_CB_HANDSHAKE_START:
|
||||
_maybeSetHostNameIndication(connection, self._hostnameBytes)
|
||||
set_tlsext_host_name(connection, self._hostnameBytes)
|
||||
elif where & SSL_CB_HANDSHAKE_DONE:
|
||||
try:
|
||||
verifyHostname(connection, self._hostnameASCII)
|
||||
|
|
@ -62,8 +77,3 @@ try:
|
|||
self._hostnameASCII, repr(e)))
|
||||
|
||||
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
|
||||
|
||||
except ImportError:
|
||||
# ImportError should not matter for older Twisted versions
|
||||
# as the above is not used in the fallback ScrapyClientContextFactory
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -6,10 +6,18 @@ from scrapy.responsetypes import responsetypes
|
|||
from scrapy.exceptions import NotConfigured
|
||||
|
||||
|
||||
ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
|
||||
|
||||
try:
|
||||
import brotli
|
||||
ACCEPTED_ENCODINGS.append(b'br')
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class HttpCompressionMiddleware(object):
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool('COMPRESSION_ENABLED'):
|
||||
|
|
@ -17,7 +25,8 @@ class HttpCompressionMiddleware(object):
|
|||
return cls()
|
||||
|
||||
def process_request(self, request, spider):
|
||||
request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
|
||||
request.headers.setdefault('Accept-Encoding',
|
||||
b",".join(ACCEPTED_ENCODINGS))
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
|
||||
|
|
@ -55,5 +64,6 @@ class HttpCompressionMiddleware(object):
|
|||
# http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
|
||||
# http://www.gzip.org/zlib/zlib_faq.html#faq38
|
||||
body = zlib.decompress(body, -15)
|
||||
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
|
||||
body = brotli.decompress(body)
|
||||
return body
|
||||
|
||||
|
|
|
|||
|
|
@ -20,23 +20,25 @@ class HttpProxyMiddleware(object):
|
|||
for type, url in getproxies().items():
|
||||
self.proxies[type] = self._get_proxy(url, type)
|
||||
|
||||
if not self.proxies:
|
||||
raise NotConfigured
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
|
||||
raise NotConfigured
|
||||
auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
|
||||
return cls(auth_encoding)
|
||||
|
||||
def _basic_auth_header(self, username, password):
|
||||
user_pass = to_bytes(
|
||||
'%s:%s' % (unquote(username), unquote(password)),
|
||||
encoding=self.auth_encoding)
|
||||
return base64.b64encode(user_pass).strip()
|
||||
|
||||
def _get_proxy(self, url, orig_type):
|
||||
proxy_type, user, password, hostport = _parse_proxy(url)
|
||||
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
|
||||
|
||||
if user:
|
||||
user_pass = to_bytes(
|
||||
'%s:%s' % (unquote(user), unquote(password)),
|
||||
encoding=self.auth_encoding)
|
||||
creds = base64.b64encode(user_pass).strip()
|
||||
creds = self._basic_auth_header(user, password)
|
||||
else:
|
||||
creds = None
|
||||
|
||||
|
|
@ -45,6 +47,15 @@ class HttpProxyMiddleware(object):
|
|||
def process_request(self, request, spider):
|
||||
# ignore if proxy is already set
|
||||
if 'proxy' in request.meta:
|
||||
if request.meta['proxy'] is None:
|
||||
return
|
||||
# extract credentials if present
|
||||
creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
|
||||
request.meta['proxy'] = proxy_url
|
||||
if creds and not request.headers.get('Proxy-Authorization'):
|
||||
request.headers['Proxy-Authorization'] = b'Basic ' + creds
|
||||
return
|
||||
elif not self.proxies:
|
||||
return
|
||||
|
||||
parsed = urlparse_cached(request)
|
||||
|
|
|
|||
|
|
@ -53,8 +53,10 @@ class BaseRedirectMiddleware(object):
|
|||
|
||||
|
||||
class RedirectMiddleware(BaseRedirectMiddleware):
|
||||
"""Handle redirection of requests based on response status and meta-refresh html tag"""
|
||||
|
||||
"""
|
||||
Handle redirection of requests based on response status
|
||||
and meta-refresh html tag.
|
||||
"""
|
||||
def process_response(self, request, response, spider):
|
||||
if (request.meta.get('dont_redirect', False) or
|
||||
response.status in getattr(spider, 'handle_httpstatus_list', []) or
|
||||
|
|
@ -92,10 +94,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
|
|||
not isinstance(response, HtmlResponse):
|
||||
return response
|
||||
|
||||
if isinstance(response, HtmlResponse):
|
||||
interval, url = get_meta_refresh(response)
|
||||
if url and interval < self._maxdelay:
|
||||
redirected = self._redirect_request_using_get(request, url)
|
||||
return self._redirect(redirected, request, spider, 'meta refresh')
|
||||
interval, url = get_meta_refresh(response)
|
||||
if url and interval < self._maxdelay:
|
||||
redirected = self._redirect_request_using_get(request, url)
|
||||
return self._redirect(redirected, request, spider, 'meta refresh')
|
||||
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class Request(object_ref):
|
|||
|
||||
def __init__(self, url, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding='utf-8', priority=0,
|
||||
dont_filter=False, errback=None):
|
||||
dont_filter=False, errback=None, flags=None):
|
||||
|
||||
self._encoding = encoding # this one has to be set first
|
||||
self.method = str(method).upper()
|
||||
|
|
@ -36,6 +36,7 @@ class Request(object_ref):
|
|||
self.dont_filter = dont_filter
|
||||
|
||||
self._meta = dict(meta) if meta else None
|
||||
self.flags = [] if flags is None else list(flags)
|
||||
|
||||
@property
|
||||
def meta(self):
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class
|
|||
See documentation in docs/topics/request-response.rst
|
||||
"""
|
||||
|
||||
import six
|
||||
from six.moves.urllib.parse import urljoin, urlencode
|
||||
|
||||
import lxml.html
|
||||
from parsel.selector import create_root_node
|
||||
import six
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.utils.python import to_bytes, is_listlike
|
||||
from scrapy.utils.response import get_base_url
|
||||
|
|
@ -51,7 +54,10 @@ class FormRequest(Request):
|
|||
|
||||
def _get_form_url(form, url):
|
||||
if url is None:
|
||||
return urljoin(form.base_url, form.action)
|
||||
action = form.get('action')
|
||||
if action is None:
|
||||
return form.base_url
|
||||
return urljoin(form.base_url, strip_html5_whitespace(action))
|
||||
return urljoin(form.base_url, url)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,9 @@ See documentation in docs/topics/request-response.rst
|
|||
"""
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.headers import Headers
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.trackref import object_ref
|
||||
from scrapy.http.common import obsolete_setter
|
||||
from scrapy.exceptions import NotSupported
|
||||
|
|
@ -101,3 +103,31 @@ class Response(object_ref):
|
|||
is text (subclasses of TextResponse).
|
||||
"""
|
||||
raise NotSupported("Response content isn't text")
|
||||
|
||||
def follow(self, url, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding='utf-8', priority=0,
|
||||
dont_filter=False, errback=None):
|
||||
# type: (...) -> Request
|
||||
"""
|
||||
Return a :class:`~.Request` instance to follow a link ``url``.
|
||||
It accepts the same arguments as ``Request.__init__`` method,
|
||||
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
|
||||
not only an absolute URL.
|
||||
|
||||
:class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
|
||||
method which supports selectors in addition to absolute/relative URLs
|
||||
and Link objects.
|
||||
"""
|
||||
if isinstance(url, Link):
|
||||
url = url.url
|
||||
url = self.urljoin(url)
|
||||
return Request(url, callback,
|
||||
method=method,
|
||||
headers=headers,
|
||||
body=body,
|
||||
cookies=cookies,
|
||||
meta=meta,
|
||||
encoding=encoding,
|
||||
priority=priority,
|
||||
dont_filter=dont_filter,
|
||||
errback=errback)
|
||||
|
|
|
|||
|
|
@ -8,8 +8,12 @@ See documentation in docs/topics/request-response.rst
|
|||
import six
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
import parsel
|
||||
from w3lib.encoding import html_to_unicode, resolve_encoding, \
|
||||
html_body_declared_encoding, http_content_type_encoding
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.utils.python import memoizemethod_noargs, to_native_str
|
||||
|
|
@ -111,8 +115,60 @@ class TextResponse(Response):
|
|||
self._cached_selector = Selector(self)
|
||||
return self._cached_selector
|
||||
|
||||
def xpath(self, query):
|
||||
return self.selector.xpath(query)
|
||||
def xpath(self, query, **kwargs):
|
||||
return self.selector.xpath(query, **kwargs)
|
||||
|
||||
def css(self, query):
|
||||
return self.selector.css(query)
|
||||
|
||||
def follow(self, url, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding=None, priority=0,
|
||||
dont_filter=False, errback=None):
|
||||
# type: (...) -> Request
|
||||
"""
|
||||
Return a :class:`~.Request` instance to follow a link ``url``.
|
||||
It accepts the same arguments as ``Request.__init__`` method,
|
||||
but ``url`` can be not only an absolute URL, but also
|
||||
|
||||
* a relative URL;
|
||||
* a scrapy.link.Link object (e.g. a link extractor result);
|
||||
* an attribute Selector (not SelectorList) - e.g.
|
||||
``response.css('a::attr(href)')[0]`` or
|
||||
``response.xpath('//img/@src')[0]``.
|
||||
* a Selector for ``<a>`` element, e.g.
|
||||
``response.css('a.my_link')[0]``.
|
||||
|
||||
See :ref:`response-follow-example` for usage examples.
|
||||
"""
|
||||
if isinstance(url, parsel.Selector):
|
||||
url = _url_from_selector(url)
|
||||
elif isinstance(url, parsel.SelectorList):
|
||||
raise ValueError("SelectorList is not supported")
|
||||
encoding = self.encoding if encoding is None else encoding
|
||||
return super(TextResponse, self).follow(url, callback,
|
||||
method=method,
|
||||
headers=headers,
|
||||
body=body,
|
||||
cookies=cookies,
|
||||
meta=meta,
|
||||
encoding=encoding,
|
||||
priority=priority,
|
||||
dont_filter=dont_filter,
|
||||
errback=errback
|
||||
)
|
||||
|
||||
|
||||
def _url_from_selector(sel):
|
||||
# type: (parsel.Selector) -> str
|
||||
if isinstance(sel.root, six.string_types):
|
||||
# e.g. ::attr(href) result
|
||||
return strip_html5_whitespace(sel.root)
|
||||
if not hasattr(sel.root, 'tag'):
|
||||
raise ValueError("Unsupported selector: %s" % sel)
|
||||
if sel.root.tag != 'a':
|
||||
raise ValueError("Only <a> elements are supported; got <%s>" %
|
||||
sel.root.tag)
|
||||
href = sel.root.get('href')
|
||||
if href is None:
|
||||
raise ValueError("<a> element has no href attribute: %s" % sel)
|
||||
return strip_html5_whitespace(href)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ class Field(dict):
|
|||
class ItemMeta(ABCMeta):
|
||||
|
||||
def __new__(mcs, class_name, bases, attrs):
|
||||
classcell = attrs.pop('__classcell__', None)
|
||||
new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
|
||||
_class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
|
||||
|
||||
|
|
@ -39,6 +40,8 @@ class ItemMeta(ABCMeta):
|
|||
|
||||
new_attrs['fields'] = fields
|
||||
new_attrs['_class'] = _class
|
||||
if classcell is not None:
|
||||
new_attrs['__classcell__'] = classcell
|
||||
return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ IGNORED_EXTENSIONS = [
|
|||
|
||||
|
||||
_re_type = type(re.compile("", 0))
|
||||
_matches = lambda url, regexs: any((r.search(url) for r in regexs))
|
||||
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'}
|
||||
|
||||
|
||||
|
|
@ -93,8 +93,8 @@ class FilteringLinkExtractor(object):
|
|||
if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
|
||||
return False
|
||||
|
||||
allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
|
||||
denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
|
||||
allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True]
|
||||
denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
|
||||
return any(allowed) and not any(denied)
|
||||
|
||||
def _process_links(self, links):
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
"""
|
||||
HTMLParser-based link extractor
|
||||
"""
|
||||
|
||||
import warnings
|
||||
import six
|
||||
from six.moves.html_parser import HTMLParser
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
|
|
@ -16,7 +16,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
|
|||
|
||||
class HtmlParserLinkExtractor(HTMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False):
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False,
|
||||
strip=True):
|
||||
HTMLParser.__init__(self)
|
||||
|
||||
warnings.warn(
|
||||
|
|
@ -29,6 +30,7 @@ class HtmlParserLinkExtractor(HTMLParser):
|
|||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding):
|
||||
self.reset()
|
||||
|
|
@ -69,6 +71,8 @@ class HtmlParserLinkExtractor(HTMLParser):
|
|||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if self.strip:
|
||||
value = strip_html5_whitespace(value)
|
||||
url = self.process_attr(value)
|
||||
link = Link(url=url)
|
||||
self.links.append(link)
|
||||
|
|
|
|||
|
|
@ -2,15 +2,16 @@
|
|||
Link extractor based on lxml.html
|
||||
"""
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse, urljoin
|
||||
from six.moves.urllib.parse import urljoin
|
||||
|
||||
import lxml.etree as etree
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list, to_native_str
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
|
||||
|
||||
# from lxml/src/lxml/html/__init__.py
|
||||
|
|
@ -27,11 +28,13 @@ def _nons(tag):
|
|||
|
||||
|
||||
class LxmlParserLinkExtractor(object):
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False):
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False,
|
||||
strip=True):
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
|
||||
def _iter_links(self, document):
|
||||
for el in document.iter(etree.Element):
|
||||
|
|
@ -49,9 +52,11 @@ class LxmlParserLinkExtractor(object):
|
|||
for el, attr, attr_val in self._iter_links(selector.root):
|
||||
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
|
||||
try:
|
||||
if self.strip:
|
||||
attr_val = strip_html5_whitespace(attr_val)
|
||||
attr_val = urljoin(base_url, attr_val)
|
||||
except ValueError:
|
||||
continue # skipping bogus links
|
||||
continue # skipping bogus links
|
||||
else:
|
||||
url = self.process_attr(attr_val)
|
||||
if url is None:
|
||||
|
|
@ -85,12 +90,13 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
tags=('a', 'area'), attrs=('href',), canonicalize=True,
|
||||
unique=True, process_value=None, deny_extensions=None, restrict_css=()):
|
||||
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
|
||||
strip=True):
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func,
|
||||
unique=unique, process=process_value)
|
||||
unique=unique, process=process_value, strip=strip)
|
||||
|
||||
super(LxmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
|
||||
allow_domains=allow_domains, deny_domains=deny_domains,
|
||||
|
|
|
|||
|
|
@ -10,9 +10,10 @@ linkre = re.compile(
|
|||
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
|
||||
re.DOTALL | re.IGNORECASE)
|
||||
|
||||
|
||||
def clean_link(link_text):
|
||||
"""Remove leading and trailing whitespace and punctuation"""
|
||||
return link_text.strip("\t\r\n '\"")
|
||||
return link_text.strip("\t\r\n '\"\x0c")
|
||||
|
||||
|
||||
class RegexLinkExtractor(SgmlLinkExtractor):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,8 @@ import warnings
|
|||
from sgmllib import SGMLParser
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
from scrapy.selector import Selector
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
|
|
@ -18,7 +19,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
|
|||
|
||||
class BaseSgmlLinkExtractor(SGMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", unique=False, process_value=None):
|
||||
def __init__(self, tag="a", attr="href", unique=False, process_value=None,
|
||||
strip=True):
|
||||
warnings.warn(
|
||||
"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
|
||||
"Please use scrapy.linkextractors.LinkExtractor",
|
||||
|
|
@ -30,6 +32,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
|
|||
self.process_value = (lambda v: v) if process_value is None else process_value
|
||||
self.current_link = None
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
|
||||
""" Do the real extraction work """
|
||||
|
|
@ -79,6 +82,8 @@ class BaseSgmlLinkExtractor(SGMLParser):
|
|||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if self.strip and value is not None:
|
||||
value = strip_html5_whitespace(value)
|
||||
url = self.process_value(value)
|
||||
if url is not None:
|
||||
link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
|
||||
|
|
@ -103,7 +108,8 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
|
|||
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True,
|
||||
process_value=None, deny_extensions=None, restrict_css=()):
|
||||
process_value=None, deny_extensions=None, restrict_css=(),
|
||||
strip=True):
|
||||
|
||||
warnings.warn(
|
||||
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
|
||||
|
|
@ -118,7 +124,7 @@ class SgmlLinkExtractor(FilteringLinkExtractor):
|
|||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
|
||||
unique=unique, process_value=process_value)
|
||||
unique=unique, process_value=process_value, strip=strip)
|
||||
|
||||
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
|
||||
allow_domains=allow_domains, deny_domains=deny_domains,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from scrapy.utils.request import referer_str
|
|||
|
||||
SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
|
||||
DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
|
||||
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s (referer: %(referer)s)%(flags)s"
|
||||
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
|
||||
|
||||
|
||||
class LogFormatter(object):
|
||||
|
|
@ -32,15 +32,17 @@ class LogFormatter(object):
|
|||
"""
|
||||
|
||||
def crawled(self, request, response, spider):
|
||||
flags = ' %s' % str(response.flags) if response.flags else ''
|
||||
request_flags = ' %s' % str(request.flags) if request.flags else ''
|
||||
response_flags = ' %s' % str(response.flags) if response.flags else ''
|
||||
return {
|
||||
'level': logging.DEBUG,
|
||||
'msg': CRAWLEDMSG,
|
||||
'args': {
|
||||
'status': response.status,
|
||||
'request': request,
|
||||
'request_flags' : request_flags,
|
||||
'referer': referer_str(request),
|
||||
'flags': flags,
|
||||
'response_flags': response_flags,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,8 +16,11 @@ class CachingThreadedResolver(ThreadedResolver):
|
|||
def getHostByName(self, name, timeout=None):
|
||||
if name in dnscache:
|
||||
return defer.succeed(dnscache[name])
|
||||
if not timeout:
|
||||
timeout = self.timeout
|
||||
# in Twisted<=16.6, getHostByName() is always called with
|
||||
# a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
|
||||
# so the input argument above is simply overridden
|
||||
# to enforce Scrapy's DNS_TIMEOUT setting's value
|
||||
timeout = (self.timeout,)
|
||||
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
|
||||
d.addCallback(self._cache_result, name)
|
||||
return d
|
||||
|
|
|
|||
|
|
@ -114,8 +114,8 @@ class BaseSettings(MutableMapping):
|
|||
"""
|
||||
Get a setting value as a boolean.
|
||||
|
||||
``1``, ``'1'``, and ``True`` return ``True``, while ``0``, ``'0'``,
|
||||
``False`` and ``None`` return ``False``.
|
||||
``1``, ``'1'``, `True`` and ``'True'`` return ``True``,
|
||||
while ``0``, ``'0'``, ``False``, ``'False'`` and ``None`` return ``False``.
|
||||
|
||||
For example, settings populated through environment variables set to
|
||||
``'0'`` will return ``False`` when using this method.
|
||||
|
|
@ -126,7 +126,17 @@ class BaseSettings(MutableMapping):
|
|||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
"""
|
||||
return bool(int(self.get(name, default)))
|
||||
got = self.get(name, default)
|
||||
try:
|
||||
return bool(int(got))
|
||||
except ValueError:
|
||||
if got in ("True", "true"):
|
||||
return True
|
||||
if got in ("False", "false"):
|
||||
return False
|
||||
raise ValueError("Supported values for boolean settings "
|
||||
"are 0/1, True/False, '0'/'1', "
|
||||
"'True'/'False' and 'true'/'false'")
|
||||
|
||||
def getint(self, name, default=0):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -161,6 +161,10 @@ FEED_EXPORTERS_BASE = {
|
|||
|
||||
FILES_STORE_S3_ACL = 'private'
|
||||
|
||||
FTP_USER = 'anonymous'
|
||||
FTP_PASSWORD = 'guest'
|
||||
FTP_PASSIVE_MODE = True
|
||||
|
||||
HTTPCACHE_ENABLED = False
|
||||
HTTPCACHE_DIR = 'httpcache'
|
||||
HTTPCACHE_IGNORE_MISSING = False
|
||||
|
|
@ -174,6 +178,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
|
|||
HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
|
||||
HTTPCACHE_GZIP = False
|
||||
|
||||
HTTPPROXY_ENABLED = True
|
||||
HTTPPROXY_AUTH_ENCODING = 'latin-1'
|
||||
|
||||
IMAGES_STORE_S3_ACL = 'private'
|
||||
|
|
@ -207,7 +212,7 @@ MEMDEBUG_ENABLED = False # enable memory debugging
|
|||
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
|
||||
|
||||
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
|
||||
MEMUSAGE_ENABLED = False
|
||||
MEMUSAGE_ENABLED = True
|
||||
MEMUSAGE_LIMIT_MB = 0
|
||||
MEMUSAGE_NOTIFY_MAIL = []
|
||||
MEMUSAGE_REPORT = False
|
||||
|
|
|
|||
|
|
@ -46,6 +46,10 @@ class HttpErrorMiddleware(object):
|
|||
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
if isinstance(exception, HttpError):
|
||||
spider.crawler.stats.inc_value('httperror/response_ignored_count')
|
||||
spider.crawler.stats.inc_value(
|
||||
'httperror/response_ignored_status_count/%s' % response.status
|
||||
)
|
||||
logger.info(
|
||||
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
|
||||
{'response': response}, extra={'spider': spider},
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from scrapy.utils.trackref import object_ref
|
|||
from scrapy.utils.url import url_is_from_spider
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.deprecate import method_is_overridden
|
||||
|
||||
|
||||
class Spider(object_ref):
|
||||
|
|
@ -66,10 +67,23 @@ class Spider(object_ref):
|
|||
crawler.signals.connect(self.close, signals.spider_closed)
|
||||
|
||||
def start_requests(self):
|
||||
for url in self.start_urls:
|
||||
yield self.make_requests_from_url(url)
|
||||
cls = self.__class__
|
||||
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
|
||||
warnings.warn(
|
||||
"Spider.make_requests_from_url method is deprecated; it "
|
||||
"won't be called in future Scrapy releases. Please "
|
||||
"override Spider.start_requests method instead (see %s.%s)." % (
|
||||
cls.__module__, cls.__name__
|
||||
),
|
||||
)
|
||||
for url in self.start_urls:
|
||||
yield self.make_requests_from_url(url)
|
||||
else:
|
||||
for url in self.start_urls:
|
||||
yield Request(url, dont_filter=True)
|
||||
|
||||
def make_requests_from_url(self, url):
|
||||
""" This method is deprecated. """
|
||||
return Request(url, dont_filter=True)
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -48,6 +48,11 @@ class CrawlSpider(Spider):
|
|||
def process_results(self, response, results):
|
||||
return results
|
||||
|
||||
def _build_request(self, rule, link):
|
||||
r = Request(url=link.url, callback=self._response_downloaded)
|
||||
r.meta.update(rule=rule, link_text=link.text)
|
||||
return r
|
||||
|
||||
def _requests_to_follow(self, response):
|
||||
if not isinstance(response, HtmlResponse):
|
||||
return
|
||||
|
|
@ -59,8 +64,7 @@ class CrawlSpider(Spider):
|
|||
links = rule.process_links(links)
|
||||
for link in links:
|
||||
seen.add(link)
|
||||
r = Request(url=link.url, callback=self._response_downloaded)
|
||||
r.meta.update(rule=n, link_text=link.text)
|
||||
r = self._build_request(n, link)
|
||||
yield rule.process_request(r)
|
||||
|
||||
def _response_downloaded(self, response):
|
||||
|
|
|
|||
|
|
@ -20,14 +20,14 @@ class ${ProjectName}SpiderMiddleware(object):
|
|||
crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
|
||||
return s
|
||||
|
||||
def process_spider_input(response, spider):
|
||||
def process_spider_input(self, response, spider):
|
||||
# Called for each response that goes through the spider
|
||||
# middleware and into the spider.
|
||||
|
||||
# Should return None or raise an exception.
|
||||
return None
|
||||
|
||||
def process_spider_output(response, result, spider):
|
||||
def process_spider_output(self, response, result, spider):
|
||||
# Called with the results returned from the Spider, after
|
||||
# it has processed the response.
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ class ${ProjectName}SpiderMiddleware(object):
|
|||
for i in result:
|
||||
yield i
|
||||
|
||||
def process_spider_exception(response, exception, spider):
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
# Called when a spider or process_spider_input() method
|
||||
# (from other spider middleware) raises an exception.
|
||||
|
||||
|
|
@ -43,7 +43,7 @@ class ${ProjectName}SpiderMiddleware(object):
|
|||
# or Item objects.
|
||||
pass
|
||||
|
||||
def process_start_requests(start_requests, spider):
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
# Called with the start requests of the spider, and works
|
||||
# similarly to the process_spider_output() method, except
|
||||
# that it doesn’t have a response associated.
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ ROBOTSTXT_OBEY = True
|
|||
# Configure item pipelines
|
||||
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
|
||||
#ITEM_PIPELINES = {
|
||||
# '$project_name.pipelines.SomePipeline': 300,
|
||||
# '$project_name.pipelines.${ProjectName}Pipeline': 300,
|
||||
#}
|
||||
|
||||
# Enable and configure the AutoThrottle extension (disabled by default)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ import scrapy
|
|||
|
||||
|
||||
class $classname(scrapy.Spider):
|
||||
name = "$name"
|
||||
allowed_domains = ["$domain"]
|
||||
name = '$name'
|
||||
allowed_domains = ['$domain']
|
||||
start_urls = ['http://$domain/']
|
||||
|
||||
def parse(self, response):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
import sys
|
||||
import numbers
|
||||
from operator import itemgetter
|
||||
|
||||
import six
|
||||
|
|
@ -34,6 +35,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
|
|||
_check_components(compdict)
|
||||
return {convert(k): v for k, v in six.iteritems(compdict)}
|
||||
|
||||
def _validate_values(compdict):
|
||||
"""Fail if a value in the components dict is not a real number or None."""
|
||||
for name, value in six.iteritems(compdict):
|
||||
if value is not None and not isinstance(value, numbers.Real):
|
||||
raise ValueError('Invalid value {} for component {}, please provide ' \
|
||||
'a real number or None instead'.format(value, name))
|
||||
|
||||
# BEGIN Backwards compatibility for old (base, custom) call signature
|
||||
if isinstance(custom, (list, tuple)):
|
||||
_check_components(custom)
|
||||
|
|
@ -43,6 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
|
|||
compdict.update(custom)
|
||||
# END Backwards compatibility
|
||||
|
||||
_validate_values(compdict)
|
||||
compdict = without_none_values(_map_keys(compdict))
|
||||
return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
|
||||
|
||||
|
|
|
|||
|
|
@ -156,3 +156,35 @@ def update_classpath(path):
|
|||
ScrapyDeprecationWarning)
|
||||
return new_path
|
||||
return path
|
||||
|
||||
|
||||
def method_is_overridden(subclass, base_class, method_name):
|
||||
"""
|
||||
Return True if a method named ``method_name`` of a ``base_class``
|
||||
is overridden in a ``subclass``.
|
||||
|
||||
>>> class Base(object):
|
||||
... def foo(self):
|
||||
... pass
|
||||
>>> class Sub1(Base):
|
||||
... pass
|
||||
>>> class Sub2(Base):
|
||||
... def foo(self):
|
||||
... pass
|
||||
>>> class Sub3(Sub1):
|
||||
... def foo(self):
|
||||
... pass
|
||||
>>> class Sub4(Sub2):
|
||||
... pass
|
||||
>>> method_is_overridden(Sub1, Base, 'foo')
|
||||
False
|
||||
>>> method_is_overridden(Sub2, Base, 'foo')
|
||||
True
|
||||
>>> method_is_overridden(Sub3, Base, 'foo')
|
||||
True
|
||||
>>> method_is_overridden(Sub4, Base, 'foo')
|
||||
True
|
||||
"""
|
||||
base_method = getattr(base_class, method_name)
|
||||
sub_method = getattr(subclass, method_name)
|
||||
return base_method.__code__ is not sub_method.__code__
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import six
|
|||
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.python import to_unicode, to_native_str
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
def request_to_dict(request, spider=None):
|
||||
|
|
@ -31,7 +32,10 @@ def request_to_dict(request, spider=None):
|
|||
'_encoding': request._encoding,
|
||||
'priority': request.priority,
|
||||
'dont_filter': request.dont_filter,
|
||||
'flags': request.flags
|
||||
}
|
||||
if type(request) is not Request:
|
||||
d['_class'] = request.__module__ + '.' + request.__class__.__name__
|
||||
return d
|
||||
|
||||
|
||||
|
|
@ -47,7 +51,8 @@ def request_from_dict(d, spider=None):
|
|||
eb = d['errback']
|
||||
if eb and spider:
|
||||
eb = _get_method(spider, eb)
|
||||
return Request(
|
||||
request_cls = load_object(d['_class']) if '_class' in d else Request
|
||||
return request_cls(
|
||||
url=to_native_str(d['url']),
|
||||
callback=cb,
|
||||
errback=eb,
|
||||
|
|
@ -58,7 +63,8 @@ def request_from_dict(d, spider=None):
|
|||
meta=d['meta'],
|
||||
encoding=d['_encoding'],
|
||||
priority=d['priority'],
|
||||
dont_filter=d['dont_filter'])
|
||||
dont_filter=d['dont_filter'],
|
||||
flags=d.get('flags'))
|
||||
|
||||
|
||||
def _find_method(obj, func):
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
:orphan:
|
||||
|
||||
Scrapy Enhancement Proposals
|
||||
============================
|
||||
|
||||
5
setup.py
5
setup.py
|
|
@ -36,19 +36,20 @@ setup(
|
|||
'Programming Language :: Python :: 3.3',
|
||||
'Programming Language :: Python :: 3.4',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Topic :: Internet :: WWW/HTTP',
|
||||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
install_requires=[
|
||||
'Twisted>=13.1.0',
|
||||
'w3lib>=1.15.0',
|
||||
'w3lib>=1.17.0',
|
||||
'queuelib',
|
||||
'lxml',
|
||||
'pyOpenSSL',
|
||||
'cssselect>=0.9',
|
||||
'six>=1.5.2',
|
||||
'parsel>=0.9.5',
|
||||
'parsel>=1.1',
|
||||
'PyDispatcher>=2.0.5',
|
||||
'service_identity',
|
||||
],
|
||||
|
|
|
|||
|
|
@ -26,9 +26,12 @@ try:
|
|||
except ImportError:
|
||||
import mock
|
||||
|
||||
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
|
||||
tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
|
||||
'sample_data')
|
||||
|
||||
|
||||
def get_testdata(*paths):
|
||||
"""Return test data"""
|
||||
path = os.path.join(tests_datadir, *paths)
|
||||
return open(path, 'rb').read()
|
||||
with open(path, 'rb') as f:
|
||||
return f.read()
|
||||
|
|
|
|||
|
|
@ -2,34 +2,18 @@ from __future__ import print_function
|
|||
import sys, time, random, os, json
|
||||
from six.moves.urllib.parse import urlencode
|
||||
from subprocess import Popen, PIPE
|
||||
|
||||
from twisted.web.server import Site, NOT_DONE_YET
|
||||
from twisted.web.resource import Resource
|
||||
from twisted.internet import reactor, defer, ssl
|
||||
from scrapy import twisted_version
|
||||
from twisted.web.test.test_webclient import PayloadResource
|
||||
from twisted.web.server import GzipEncoderFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper
|
||||
from twisted.internet import reactor, ssl
|
||||
from twisted.internet.task import deferLater
|
||||
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
|
||||
|
||||
if twisted_version < (11, 0, 0):
|
||||
def deferLater(clock, delay, func, *args, **kw):
|
||||
def _cancel_method():
|
||||
_cancel_cb(None)
|
||||
d.errback(Exception())
|
||||
|
||||
def _cancel_cb(result):
|
||||
if cl.active():
|
||||
cl.cancel()
|
||||
return result
|
||||
|
||||
d = defer.Deferred()
|
||||
d.cancel = _cancel_method
|
||||
d.addCallback(lambda ignored: func(*args, **kw))
|
||||
d.addBoth(_cancel_cb)
|
||||
cl = clock.callLater(delay, d.callback, None)
|
||||
return d
|
||||
else:
|
||||
from twisted.internet.task import deferLater
|
||||
|
||||
|
||||
def getarg(request, name, default=None, type=None):
|
||||
if name in request.args:
|
||||
value = request.args[name][0]
|
||||
|
|
@ -174,13 +158,8 @@ class Root(Resource):
|
|||
self.putChild(b"drop", Drop())
|
||||
self.putChild(b"raw", Raw())
|
||||
self.putChild(b"echo", Echo())
|
||||
|
||||
if twisted_version > (12, 3, 0):
|
||||
from twisted.web.test.test_webclient import PayloadResource
|
||||
from twisted.web.server import GzipEncoderFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper
|
||||
self.putChild(b"payload", PayloadResource())
|
||||
self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
|
||||
self.putChild(b"payload", PayloadResource())
|
||||
self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
|
||||
|
||||
def getChild(self, name, request):
|
||||
return self
|
||||
|
|
|
|||
|
|
@ -8,3 +8,4 @@ botocore
|
|||
# optional for shell wrapper tests
|
||||
bpython
|
||||
ipython
|
||||
brotlipy
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ pytest==2.9.2
|
|||
pytest-twisted
|
||||
pytest-cov==2.2.1
|
||||
jmespath
|
||||
brotlipy
|
||||
testfixtures
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -13,6 +13,7 @@
|
|||
<a href='sample3.html'>sample 3 repetition</a>
|
||||
<a href='http://www.google.com/something'></a>
|
||||
<a href='http://example.com/innertag.html'><b>inner</b> tag</a>
|
||||
<a href=' page 4.html '>href with whitespaces</a>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -170,10 +170,7 @@ class DuplicateStartRequestsSpider(Spider):
|
|||
for i in range(0, self.distinct_urls):
|
||||
for j in range(0, self.dupe_factor):
|
||||
url = "http://localhost:8998/echo?headers=1&body=test%d" % i
|
||||
yield self.make_requests_from_url(url)
|
||||
|
||||
def make_requests_from_url(self, url):
|
||||
return Request(url, dont_filter=self.dont_filter)
|
||||
yield Request(url, dont_filter=self.dont_filter)
|
||||
|
||||
def __init__(self, url="http://localhost:8998", *args, **kwargs):
|
||||
super(DuplicateStartRequestsSpider, self).__init__(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import json
|
||||
import socket
|
||||
import logging
|
||||
|
||||
from testfixtures import LogCapture
|
||||
|
|
@ -9,7 +8,6 @@ from twisted.trial.unittest import TestCase
|
|||
from scrapy.http import Request
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.python import to_unicode
|
||||
from tests import mock
|
||||
from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
|
||||
BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
|
||||
from tests.mockserver import MockServer
|
||||
|
|
@ -91,12 +89,11 @@ class CrawlTestCase(TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_retry_dns_error(self):
|
||||
with mock.patch('socket.gethostbyname',
|
||||
side_effect=socket.gaierror(-5, 'No address associated with hostname')):
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
yield crawler.crawl("http://example.com/")
|
||||
self._assert_retried(l)
|
||||
crawler = self.runner.create_crawler(SimpleSpider)
|
||||
with LogCapture() as l:
|
||||
# try to fetch the homepage of a non-existent domain
|
||||
yield crawler.crawl("http://dns.resolution.invalid./")
|
||||
self._assert_retried(l)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_start_requests_bug_before_yield(self):
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import os
|
||||
import six
|
||||
import contextlib
|
||||
import shutil
|
||||
try:
|
||||
from unittest import mock
|
||||
except ImportError:
|
||||
import mock
|
||||
import shutil
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
|
|
@ -17,7 +19,6 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
|
|||
from twisted.cred import portal, checkers, credentials
|
||||
from w3lib.url import path_to_file_uri
|
||||
|
||||
from scrapy import twisted_version
|
||||
from scrapy.core.downloader.handlers import DownloadHandlers
|
||||
from scrapy.core.downloader.handlers.file import FileDownloadHandler
|
||||
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
|
||||
|
|
@ -85,11 +86,13 @@ class FileTestCase(unittest.TestCase):
|
|||
|
||||
def setUp(self):
|
||||
self.tmpname = self.mktemp()
|
||||
fd = open(self.tmpname + '^', 'w')
|
||||
fd.write('0123456789')
|
||||
fd.close()
|
||||
with open(self.tmpname + '^', 'w') as f:
|
||||
f.write('0123456789')
|
||||
self.download_request = FileDownloadHandler(Settings()).download_request
|
||||
|
||||
def tearDown(self):
|
||||
os.unlink(self.tmpname + '^')
|
||||
|
||||
def test_download(self):
|
||||
def _test(response):
|
||||
self.assertEquals(response.url, request.url)
|
||||
|
|
@ -135,10 +138,10 @@ class HttpTestCase(unittest.TestCase):
|
|||
certfile = 'keys/cert.pem'
|
||||
|
||||
def setUp(self):
|
||||
name = self.mktemp()
|
||||
os.mkdir(name)
|
||||
FilePath(name).child("file").setContent(b"0123456789")
|
||||
r = static.File(name)
|
||||
self.tmpname = self.mktemp()
|
||||
os.mkdir(self.tmpname)
|
||||
FilePath(self.tmpname).child("file").setContent(b"0123456789")
|
||||
r = static.File(self.tmpname)
|
||||
r.putChild(b"redirect", util.Redirect(b"/file"))
|
||||
r.putChild(b"wait", ForeverTakingResource())
|
||||
r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
|
||||
|
|
@ -166,6 +169,7 @@ class HttpTestCase(unittest.TestCase):
|
|||
yield self.port.stopListening()
|
||||
if hasattr(self.download_handler, 'close'):
|
||||
yield self.download_handler.close()
|
||||
shutil.rmtree(self.tmpname)
|
||||
|
||||
def getURL(self, path):
|
||||
return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
|
||||
|
|
@ -281,8 +285,6 @@ class Https10TestCase(Http10TestCase):
|
|||
class Http11TestCase(HttpTestCase):
|
||||
"""HTTP 1.1 test case"""
|
||||
download_handler_cls = HTTP11DownloadHandler
|
||||
if twisted_version < (11, 1, 0):
|
||||
skip = 'HTTP1.1 not supported in twisted < 11.1.0'
|
||||
|
||||
def test_download_without_maxsize_limit(self):
|
||||
request = Request(self.getURL('file'))
|
||||
|
|
@ -366,8 +368,6 @@ class Https11InvalidDNSId(Https11TestCase):
|
|||
|
||||
class Http11MockServerTestCase(unittest.TestCase):
|
||||
"""HTTP 1.1 test case with MockServer"""
|
||||
if twisted_version < (11, 1, 0):
|
||||
skip = 'HTTP1.1 not supported in twisted < 11.1.0'
|
||||
|
||||
def setUp(self):
|
||||
self.mockserver = MockServer()
|
||||
|
|
@ -396,31 +396,27 @@ class Http11MockServerTestCase(unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_download_gzip_response(self):
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
body = b'1' * 100 # PayloadResource requires body length to be 100
|
||||
request = Request('http://localhost:8998/payload', method='POST',
|
||||
body=body, meta={'download_maxsize': 50})
|
||||
yield crawler.crawl(seed=request)
|
||||
failure = crawler.spider.meta['failure']
|
||||
# download_maxsize < 100, hence the CancelledError
|
||||
self.assertIsInstance(failure.value, defer.CancelledError)
|
||||
|
||||
if twisted_version > (12, 3, 0):
|
||||
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
body = b'1'*100 # PayloadResource requires body length to be 100
|
||||
request = Request('http://localhost:8998/payload', method='POST', body=body, meta={'download_maxsize': 50})
|
||||
if six.PY2:
|
||||
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
|
||||
request = request.replace(url='http://localhost:8998/xpayload')
|
||||
yield crawler.crawl(seed=request)
|
||||
failure = crawler.spider.meta['failure']
|
||||
# download_maxsize < 100, hence the CancelledError
|
||||
self.assertIsInstance(failure.value, defer.CancelledError)
|
||||
|
||||
if six.PY2:
|
||||
request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate')
|
||||
request = request.replace(url='http://localhost:8998/xpayload')
|
||||
yield crawler.crawl(seed=request)
|
||||
# download_maxsize = 50 is enough for the gzipped response
|
||||
failure = crawler.spider.meta.get('failure')
|
||||
self.assertTrue(failure == None)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertTrue(reason, 'finished')
|
||||
else:
|
||||
# See issue https://twistedmatrix.com/trac/ticket/8175
|
||||
raise unittest.SkipTest("xpayload only enabled for PY2")
|
||||
# download_maxsize = 50 is enough for the gzipped response
|
||||
failure = crawler.spider.meta.get('failure')
|
||||
self.assertTrue(failure == None)
|
||||
reason = crawler.spider.meta['close_reason']
|
||||
self.assertTrue(reason, 'finished')
|
||||
else:
|
||||
raise unittest.SkipTest("xpayload and payload endpoint only enabled for twisted > 12.3.0")
|
||||
# See issue https://twistedmatrix.com/trac/ticket/8175
|
||||
raise unittest.SkipTest("xpayload only enabled for PY2")
|
||||
|
||||
|
||||
class UriResource(resource.Resource):
|
||||
|
|
@ -500,8 +496,6 @@ class Http10ProxyTestCase(HttpProxyTestCase):
|
|||
|
||||
class Http11ProxyTestCase(HttpProxyTestCase):
|
||||
download_handler_cls = HTTP11DownloadHandler
|
||||
if twisted_version < (11, 1, 0):
|
||||
skip = 'HTTP1.1 not supported in twisted < 11.1.0'
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_download_with_proxy_https_timeout(self):
|
||||
|
|
@ -687,13 +681,12 @@ class S3TestCase(unittest.TestCase):
|
|||
b'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
|
||||
|
||||
|
||||
class FTPTestCase(unittest.TestCase):
|
||||
class BaseFTPTestCase(unittest.TestCase):
|
||||
|
||||
username = "scrapy"
|
||||
password = "passwd"
|
||||
req_meta = {"ftp_user": username, "ftp_password": password}
|
||||
|
||||
if twisted_version < (10, 2, 0):
|
||||
skip = "Twisted pre 10.2.0 doesn't allow to set home path other than /home"
|
||||
if six.PY3:
|
||||
skip = "Twisted missing ftp support for PY3"
|
||||
|
||||
|
|
@ -722,6 +715,9 @@ class FTPTestCase(unittest.TestCase):
|
|||
self.download_handler = FTPDownloadHandler(Settings())
|
||||
self.addCleanup(self.port.stopListening)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.directory)
|
||||
|
||||
def _add_test_callbacks(self, deferred, callback=None, errback=None):
|
||||
def _clean(data):
|
||||
self.download_handler.client.transport.loseConnection()
|
||||
|
|
@ -735,7 +731,7 @@ class FTPTestCase(unittest.TestCase):
|
|||
|
||||
def test_ftp_download_success(self):
|
||||
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
|
||||
meta={"ftp_user": self.username, "ftp_password": self.password})
|
||||
meta=self.req_meta)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
def _test(r):
|
||||
|
|
@ -747,7 +743,7 @@ class FTPTestCase(unittest.TestCase):
|
|||
def test_ftp_download_path_with_spaces(self):
|
||||
request = Request(
|
||||
url="ftp://127.0.0.1:%s/file with spaces.txt" % self.portNum,
|
||||
meta={"ftp_user": self.username, "ftp_password": self.password}
|
||||
meta=self.req_meta
|
||||
)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
|
|
@ -759,7 +755,7 @@ class FTPTestCase(unittest.TestCase):
|
|||
|
||||
def test_ftp_download_notexist(self):
|
||||
request = Request(url="ftp://127.0.0.1:%s/notexist.txt" % self.portNum,
|
||||
meta={"ftp_user": self.username, "ftp_password": self.password})
|
||||
meta=self.req_meta)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
def _test(r):
|
||||
|
|
@ -768,8 +764,10 @@ class FTPTestCase(unittest.TestCase):
|
|||
|
||||
def test_ftp_local_filename(self):
|
||||
local_fname = "/tmp/file.txt"
|
||||
meta = {"ftp_local_filename": local_fname}
|
||||
meta.update(self.req_meta)
|
||||
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
|
||||
meta={"ftp_user": self.username, "ftp_password": self.password, "ftp_local_filename": local_fname})
|
||||
meta=meta)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
def _test(r):
|
||||
|
|
@ -781,13 +779,52 @@ class FTPTestCase(unittest.TestCase):
|
|||
os.remove(local_fname)
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
|
||||
class FTPTestCase(BaseFTPTestCase):
|
||||
|
||||
def test_invalid_credentials(self):
|
||||
from twisted.protocols.ftp import ConnectionLost
|
||||
|
||||
meta = dict(self.req_meta)
|
||||
meta.update({"ftp_password": 'invalid'})
|
||||
request = Request(url="ftp://127.0.0.1:%s/file.txt" % self.portNum,
|
||||
meta={"ftp_user": self.username, "ftp_password": 'invalid'})
|
||||
meta=meta)
|
||||
d = self.download_handler.download_request(request, None)
|
||||
|
||||
def _test(r):
|
||||
self.assertEqual(r.type, ConnectionLost)
|
||||
return self._add_test_callbacks(d, errback=_test)
|
||||
|
||||
|
||||
class AnonymousFTPTestCase(BaseFTPTestCase):
|
||||
|
||||
username = "anonymous"
|
||||
req_meta = {}
|
||||
|
||||
def setUp(self):
|
||||
from twisted.protocols.ftp import FTPRealm, FTPFactory
|
||||
from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
|
||||
|
||||
# setup dir and test file
|
||||
self.directory = self.mktemp()
|
||||
os.mkdir(self.directory)
|
||||
|
||||
fp = FilePath(self.directory)
|
||||
fp.child('file.txt').setContent("I have the power!")
|
||||
fp.child('file with spaces.txt').setContent("Moooooooooo power!")
|
||||
|
||||
# setup server for anonymous access
|
||||
realm = FTPRealm(anonymousRoot=self.directory)
|
||||
p = portal.Portal(realm)
|
||||
p.registerChecker(checkers.AllowAnonymousAccess(),
|
||||
credentials.IAnonymous)
|
||||
|
||||
self.factory = FTPFactory(portal=p,
|
||||
userAnonymous=self.username)
|
||||
self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")
|
||||
self.portNum = self.port.getHost().port
|
||||
self.download_handler = FTPDownloadHandler(Settings())
|
||||
self.addCleanup(self.port.stopListening)
|
||||
|
||||
def tearDown(self):
|
||||
shutil.rmtree(self.directory)
|
||||
|
|
|
|||
|
|
@ -1,11 +1,12 @@
|
|||
from io import BytesIO
|
||||
from unittest import TestCase
|
||||
from os.path import join, abspath, dirname
|
||||
from unittest import TestCase, SkipTest
|
||||
from os.path import join
|
||||
from gzip import GzipFile
|
||||
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.http import Response, Request, HtmlResponse
|
||||
from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware
|
||||
from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
|
||||
ACCEPTED_ENCODINGS
|
||||
from tests import tests_datadir
|
||||
from w3lib.encoding import resolve_encoding
|
||||
|
||||
|
|
@ -17,8 +18,10 @@ FORMAT = {
|
|||
'x-gzip': ('html-gzip.bin', 'gzip'),
|
||||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br')
|
||||
}
|
||||
|
||||
|
||||
class HttpCompressionTest(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
|
|
@ -50,7 +53,8 @@ class HttpCompressionTest(TestCase):
|
|||
request = Request('http://scrapytest.org')
|
||||
assert 'Accept-Encoding' not in request.headers
|
||||
self.mw.process_request(request, self.spider)
|
||||
self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
|
||||
self.assertEqual(request.headers.get('Accept-Encoding'),
|
||||
b','.join(ACCEPTED_ENCODINGS))
|
||||
|
||||
def test_process_response_gzip(self):
|
||||
response = self._getresponse('gzip')
|
||||
|
|
@ -62,6 +66,19 @@ class HttpCompressionTest(TestCase):
|
|||
assert newresponse.body.startswith(b'<!DOCTYPE')
|
||||
assert 'Content-Encoding' not in newresponse.headers
|
||||
|
||||
def test_process_response_br(self):
|
||||
try:
|
||||
import brotli
|
||||
except ImportError:
|
||||
raise SkipTest("no brotli")
|
||||
response = self._getresponse('br')
|
||||
request = response.request
|
||||
self.assertEqual(response.headers['Content-Encoding'], b'br')
|
||||
newresponse = self.mw.process_response(request, response, self.spider)
|
||||
assert newresponse is not response
|
||||
assert newresponse.body.startswith(b"<!DOCTYPE")
|
||||
assert 'Content-Encoding' not in newresponse.headers
|
||||
|
||||
def test_process_response_rawdeflate(self):
|
||||
response = self._getresponse('rawdeflate')
|
||||
request = response.request
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
import os
|
||||
import sys
|
||||
from functools import partial
|
||||
from twisted.trial.unittest import TestCase, SkipTest
|
||||
|
||||
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.settings import Settings
|
||||
|
||||
spider = Spider('foo')
|
||||
|
||||
|
|
@ -20,9 +23,10 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
def tearDown(self):
|
||||
os.environ = self._oldenv
|
||||
|
||||
def test_no_proxies(self):
|
||||
os.environ = {}
|
||||
self.assertRaises(NotConfigured, HttpProxyMiddleware)
|
||||
def test_not_enabled(self):
|
||||
settings = Settings({'HTTPPROXY_ENABLED': False})
|
||||
crawler = Crawler(spider, settings)
|
||||
self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
|
||||
|
||||
def test_no_enviroment_proxies(self):
|
||||
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
|
||||
|
|
@ -47,6 +51,13 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
self.assertEquals(req.url, url)
|
||||
self.assertEquals(req.meta.get('proxy'), proxy)
|
||||
|
||||
def test_proxy_precedence_meta(self):
|
||||
os.environ['http_proxy'] = 'https://proxy.com'
|
||||
mw = HttpProxyMiddleware()
|
||||
req = Request('http://scrapytest.org', meta={'proxy': 'https://new.proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://new.proxy:3128'})
|
||||
|
||||
def test_proxy_auth(self):
|
||||
os.environ['http_proxy'] = 'https://user:pass@proxy:3128'
|
||||
mw = HttpProxyMiddleware()
|
||||
|
|
@ -54,6 +65,11 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
|
||||
# proxy from request.meta
|
||||
req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
|
||||
|
||||
def test_proxy_auth_empty_passwd(self):
|
||||
os.environ['http_proxy'] = 'https://user:@proxy:3128'
|
||||
|
|
@ -62,6 +78,11 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
|
||||
# proxy from request.meta
|
||||
req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
|
||||
|
||||
def test_proxy_auth_encoding(self):
|
||||
# utf-8 encoding
|
||||
|
|
@ -72,6 +93,12 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
|
||||
|
||||
# proxy from request.meta
|
||||
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
|
||||
|
||||
# default latin-1 encoding
|
||||
mw = HttpProxyMiddleware(auth_encoding='latin-1')
|
||||
req = Request('http://scrapytest.org')
|
||||
|
|
@ -79,15 +106,21 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
|
||||
|
||||
# proxy from request.meta, latin-1 encoding
|
||||
req = Request('http://scrapytest.org', meta={'proxy': u'https://\u00FCser:pass@proxy:3128'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
|
||||
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
|
||||
|
||||
def test_proxy_already_seted(self):
|
||||
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
|
||||
os.environ['http_proxy'] = 'https://proxy.for.http:3128'
|
||||
mw = HttpProxyMiddleware()
|
||||
req = Request('http://noproxy.com', meta={'proxy': None})
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'proxy' in req.meta and req.meta['proxy'] is None
|
||||
|
||||
def test_no_proxy(self):
|
||||
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
|
||||
os.environ['http_proxy'] = 'https://proxy.for.http:3128'
|
||||
mw = HttpProxyMiddleware()
|
||||
|
||||
os.environ['no_proxy'] = '*'
|
||||
|
|
@ -104,3 +137,9 @@ class TestDefaultHeadersMiddleware(TestCase):
|
|||
req = Request('http://noproxy.com')
|
||||
assert mw.process_request(req, spider) is None
|
||||
assert 'proxy' not in req.meta
|
||||
|
||||
# proxy from meta['proxy'] takes precedence
|
||||
os.environ['no_proxy'] = '*'
|
||||
req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEquals(req.meta, {'proxy': 'http://proxy.com'})
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from twisted.internet.error import TimeoutError, DNSLookupError, \
|
|||
ConnectionLost, TCPTimedOutError
|
||||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy import twisted_version
|
||||
from scrapy.downloadermiddlewares.retry import RetryMiddleware
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.http import Request, Response
|
||||
|
|
@ -74,9 +73,7 @@ class RetryTest(unittest.TestCase):
|
|||
def test_twistederrors(self):
|
||||
exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,
|
||||
DNSLookupError, ConnectionRefusedError, ConnectionDone,
|
||||
ConnectError, ConnectionLost]
|
||||
if twisted_version >= (11, 1, 0): # http11 available
|
||||
exceptions.append(ResponseFailed)
|
||||
ConnectError, ConnectionLost, ResponseFailed]
|
||||
|
||||
for exc in exceptions:
|
||||
req = Request('http://www.scrapytest.org/%s' % exc.__name__)
|
||||
|
|
|
|||
|
|
@ -66,8 +66,8 @@ class TestSpider(Spider):
|
|||
|
||||
|
||||
class TestDupeFilterSpider(TestSpider):
|
||||
def make_requests_from_url(self, url):
|
||||
return Request(url) # dont_filter=False
|
||||
def start_requests(self):
|
||||
return (Request(url) for url in self.start_urls) # no dont_filter=True
|
||||
|
||||
|
||||
class DictItemsSpider(TestSpider):
|
||||
|
|
|
|||
|
|
@ -57,8 +57,11 @@ class FileFeedStorageTest(unittest.TestCase):
|
|||
file.write(b"content")
|
||||
yield storage.store(file)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"content")
|
||||
try:
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"content")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
class FTPFeedStorageTest(unittest.TestCase):
|
||||
|
|
@ -79,12 +82,15 @@ class FTPFeedStorageTest(unittest.TestCase):
|
|||
file.write(b"content")
|
||||
yield storage.store(file)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"content")
|
||||
# again, to check s3 objects are overwritten
|
||||
yield storage.store(BytesIO(b"new content"))
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"new content")
|
||||
try:
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"content")
|
||||
# again, to check s3 objects are overwritten
|
||||
yield storage.store(BytesIO(b"new content"))
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"new content")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
class BlockingFeedStorageTest(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -556,7 +556,6 @@ class FormRequestTest(RequestTest):
|
|||
fs = _qs(req, to_unicode=True, encoding='latin1')
|
||||
self.assertTrue(fs[u'price in \u00a5'])
|
||||
|
||||
|
||||
def test_from_response_multiple_forms_clickdata(self):
|
||||
response = _buildresponse(
|
||||
"""<form name="form1">
|
||||
|
|
@ -989,7 +988,7 @@ class FormRequestTest(RequestTest):
|
|||
"""
|
||||
<html>
|
||||
<head>
|
||||
<base href="http://b.com/">
|
||||
<base href=" http://b.com/">
|
||||
</head>
|
||||
<body>
|
||||
<form action="test_form">
|
||||
|
|
@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest):
|
|||
req = self.request_class.from_response(response)
|
||||
self.assertEqual(req.url, 'http://b.com/test_form')
|
||||
|
||||
def test_spaces_in_action(self):
|
||||
resp = _buildresponse('<body><form action=" path\n"></form></body>')
|
||||
req = self.request_class.from_response(resp)
|
||||
self.assertEqual(req.url, 'http://example.com/path')
|
||||
|
||||
def test_from_response_css(self):
|
||||
response = _buildresponse(
|
||||
"""<form action="post.php" method="POST">
|
||||
|
|
@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest):
|
|||
self.assertRaises(ValueError, self.request_class.from_response,
|
||||
response, formcss="input[name='abc']")
|
||||
|
||||
|
||||
def _buildresponse(body, **kwargs):
|
||||
kwargs.setdefault('body', body)
|
||||
kwargs.setdefault('url', 'http://example.com')
|
||||
kwargs.setdefault('encoding', 'utf-8')
|
||||
return HtmlResponse(**kwargs)
|
||||
|
||||
|
||||
def _qs(req, encoding='utf-8', to_unicode=False):
|
||||
if req.method == 'POST':
|
||||
qs = req.body
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
|
||||
import six
|
||||
|
|
@ -8,6 +9,8 @@ from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
|
|||
from scrapy.selector import Selector
|
||||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.exceptions import NotSupported
|
||||
from scrapy.link import Link
|
||||
from tests import get_testdata
|
||||
|
||||
|
||||
class BaseResponseTest(unittest.TestCase):
|
||||
|
|
@ -140,6 +143,38 @@ class BaseResponseTest(unittest.TestCase):
|
|||
r.css('body')
|
||||
r.xpath('//body')
|
||||
|
||||
def test_follow_url_absolute(self):
|
||||
self._assert_followed_url('http://foo.example.com',
|
||||
'http://foo.example.com')
|
||||
|
||||
def test_follow_url_relative(self):
|
||||
self._assert_followed_url('foo',
|
||||
'http://example.com/foo')
|
||||
|
||||
def test_follow_link(self):
|
||||
self._assert_followed_url(Link('http://example.com/foo'),
|
||||
'http://example.com/foo')
|
||||
|
||||
def test_follow_whitespace_url(self):
|
||||
self._assert_followed_url('foo ',
|
||||
'http://example.com/foo%20')
|
||||
|
||||
def test_follow_whitespace_link(self):
|
||||
self._assert_followed_url(Link('http://example.com/foo '),
|
||||
'http://example.com/foo%20')
|
||||
|
||||
def _assert_followed_url(self, follow_obj, target_url, response=None):
|
||||
if response is None:
|
||||
response = self._links_response()
|
||||
req = response.follow(follow_obj)
|
||||
self.assertEqual(req.url, target_url)
|
||||
return req
|
||||
|
||||
def _links_response(self):
|
||||
body = get_testdata('link_extractor', 'sgml_linkextractor.html')
|
||||
resp = self.response_class('http://example.com/index', body=body)
|
||||
return resp
|
||||
|
||||
|
||||
class TextResponseTest(BaseResponseTest):
|
||||
|
||||
|
|
@ -320,6 +355,20 @@ class TextResponseTest(BaseResponseTest):
|
|||
response.selector.css("title::text").extract(),
|
||||
)
|
||||
|
||||
def test_selector_shortcuts_kwargs(self):
|
||||
body = b"<html><head><title>Some page</title><body><p class=\"content\">A nice paragraph.</p></body></html>"
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("normalize-space(//p[@class=$pclass])", pclass="content").extract(),
|
||||
response.xpath("normalize-space(//p[@class=\"content\"])").extract(),
|
||||
)
|
||||
self.assertEqual(
|
||||
response.xpath("//title[count(following::p[@class=$pclass])=$pcount]/text()",
|
||||
pclass="content", pcount=1).extract(),
|
||||
response.xpath("//title[count(following::p[@class=\"content\"])=1]/text()").extract(),
|
||||
)
|
||||
|
||||
def test_urljoin_with_base_url(self):
|
||||
"""Test urljoin shortcut which also evaluates base-url through get_base_url()."""
|
||||
body = b'<html><body><base href="https://example.net"></body></html>'
|
||||
|
|
@ -337,6 +386,89 @@ class TextResponseTest(BaseResponseTest):
|
|||
absolute = 'http://www.example.com/elsewhere/test'
|
||||
self.assertEqual(joined, absolute)
|
||||
|
||||
def test_follow_selector(self):
|
||||
resp = self._links_response()
|
||||
urls = [
|
||||
'http://example.com/sample2.html',
|
||||
'http://example.com/sample3.html',
|
||||
'http://example.com/sample3.html',
|
||||
'http://www.google.com/something',
|
||||
'http://example.com/innertag.html'
|
||||
]
|
||||
|
||||
# select <a> elements
|
||||
for sellist in [resp.css('a'), resp.xpath('//a')]:
|
||||
for sel, url in zip(sellist, urls):
|
||||
self._assert_followed_url(sel, url, response=resp)
|
||||
|
||||
# href attributes should work
|
||||
for sellist in [resp.css('a::attr(href)'), resp.xpath('//a/@href')]:
|
||||
for sel, url in zip(sellist, urls):
|
||||
self._assert_followed_url(sel, url, response=resp)
|
||||
|
||||
# non-a elements are not supported
|
||||
self.assertRaises(ValueError, resp.follow, resp.css('div')[0])
|
||||
|
||||
def test_follow_selector_list(self):
|
||||
resp = self._links_response()
|
||||
self.assertRaisesRegexp(ValueError, 'SelectorList',
|
||||
resp.follow, resp.css('a'))
|
||||
|
||||
def test_follow_selector_invalid(self):
|
||||
resp = self._links_response()
|
||||
self.assertRaisesRegexp(ValueError, 'Unsupported',
|
||||
resp.follow, resp.xpath('count(//div)')[0])
|
||||
|
||||
def test_follow_selector_attribute(self):
|
||||
resp = self._links_response()
|
||||
for src in resp.css('img::attr(src)'):
|
||||
self._assert_followed_url(src, 'http://example.com/sample2.jpg')
|
||||
|
||||
def test_follow_selector_no_href(self):
|
||||
resp = self.response_class(
|
||||
url='http://example.com',
|
||||
body=b'<html><body><a name=123>click me</a></body></html>',
|
||||
)
|
||||
self.assertRaisesRegexp(ValueError, 'no href',
|
||||
resp.follow, resp.css('a')[0])
|
||||
|
||||
def test_follow_whitespace_selector(self):
|
||||
resp = self.response_class(
|
||||
'http://example.com',
|
||||
body=b'''<html><body><a href=" foo\n">click me</a></body></html>'''
|
||||
)
|
||||
self._assert_followed_url(resp.css('a')[0],
|
||||
'http://example.com/foo',
|
||||
response=resp)
|
||||
self._assert_followed_url(resp.css('a::attr(href)')[0],
|
||||
'http://example.com/foo',
|
||||
response=resp)
|
||||
|
||||
def test_follow_encoding(self):
|
||||
resp1 = self.response_class(
|
||||
'http://example.com',
|
||||
encoding='utf8',
|
||||
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('utf8')
|
||||
)
|
||||
req = self._assert_followed_url(
|
||||
resp1.css('a')[0],
|
||||
'http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82',
|
||||
response=resp1,
|
||||
)
|
||||
self.assertEqual(req.encoding, 'utf8')
|
||||
|
||||
resp2 = self.response_class(
|
||||
'http://example.com',
|
||||
encoding='cp1251',
|
||||
body=u'<html><body><a href="foo?привет">click me</a></body></html>'.encode('cp1251')
|
||||
)
|
||||
req = self._assert_followed_url(
|
||||
resp2.css('a')[0],
|
||||
'http://example.com/foo?%EF%F0%E8%E2%E5%F2',
|
||||
response=resp2,
|
||||
)
|
||||
self.assertEqual(req.encoding, 'cp1251')
|
||||
|
||||
|
||||
class HtmlResponseTest(TextResponseTest):
|
||||
|
||||
|
|
@ -428,3 +560,21 @@ class XmlResponseTest(TextResponseTest):
|
|||
response.xpath("//elem/text()").extract(),
|
||||
response.selector.xpath("//elem/text()").extract(),
|
||||
)
|
||||
|
||||
def test_selector_shortcuts_kwargs(self):
|
||||
body = b'''<?xml version="1.0" encoding="utf-8"?>
|
||||
<xml xmlns:somens="http://scrapy.org">
|
||||
<somens:elem>value</somens:elem>
|
||||
</xml>'''
|
||||
response = self.response_class("http://www.example.com", body=body)
|
||||
|
||||
self.assertEqual(
|
||||
response.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
|
||||
response.selector.xpath("//s:elem/text()", namespaces={'s': 'http://scrapy.org'}).extract(),
|
||||
)
|
||||
|
||||
response.selector.register_namespace('s2', 'http://scrapy.org')
|
||||
self.assertEqual(
|
||||
response.xpath("//s1:elem/text()", namespaces={'s1': 'http://scrapy.org'}).extract(),
|
||||
response.selector.xpath("//s2:elem/text()").extract(),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,14 @@
|
|||
import sys
|
||||
import unittest
|
||||
|
||||
from scrapy.item import Item, Field
|
||||
import six
|
||||
|
||||
from scrapy.item import ABCMeta, Item, ItemMeta, Field
|
||||
from tests import mock
|
||||
|
||||
|
||||
PY36_PLUS = (sys.version_info.major >= 3) and (sys.version_info.minor >= 6)
|
||||
|
||||
|
||||
class ItemTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -244,5 +250,49 @@ class ItemTest(unittest.TestCase):
|
|||
self.assertNotEqual(item['name'], copied_item['name'])
|
||||
|
||||
|
||||
class ItemMetaTest(unittest.TestCase):
|
||||
|
||||
def test_new_method_propagates_classcell(self):
|
||||
new_mock = mock.Mock(side_effect=ABCMeta.__new__)
|
||||
base = ItemMeta.__bases__[0]
|
||||
|
||||
with mock.patch.object(base, '__new__', new_mock):
|
||||
|
||||
class MyItem(Item):
|
||||
if not PY36_PLUS:
|
||||
# This attribute is an internal attribute in Python 3.6+
|
||||
# and must be propagated properly. See
|
||||
# https://docs.python.org/3.6/reference/datamodel.html#creating-the-class-object
|
||||
# In <3.6, we add a dummy attribute just to ensure the
|
||||
# __new__ method propagates it correctly.
|
||||
__classcell__ = object()
|
||||
|
||||
def f(self):
|
||||
# For rationale of this see:
|
||||
# https://github.com/python/cpython/blob/ee1a81b77444c6715cbe610e951c655b6adab88b/Lib/test/test_super.py#L222
|
||||
return __class__
|
||||
|
||||
MyItem()
|
||||
|
||||
(first_call, second_call) = new_mock.call_args_list[-2:]
|
||||
|
||||
mcs, class_name, bases, attrs = first_call[0]
|
||||
assert '__classcell__' not in attrs
|
||||
mcs, class_name, bases, attrs = second_call[0]
|
||||
assert '__classcell__' in attrs
|
||||
|
||||
|
||||
class ItemMetaClassCellRegression(unittest.TestCase):
|
||||
|
||||
def test_item_meta_classcell_regression(self):
|
||||
class MyItem(six.with_metaclass(ItemMeta, Item)):
|
||||
def __init__(self, *args, **kwargs):
|
||||
# This call to super() trigger the __classcell__ propagation
|
||||
# requirement. When not done properly raises an error:
|
||||
# TypeError: __class__ set to <class '__main__.MyItem'>
|
||||
# defining 'MyItem' as <class '__main__.MyItem'>
|
||||
super(MyItem, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ class Base:
|
|||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
|
||||
])
|
||||
|
||||
def test_extract_filter_allow(self):
|
||||
|
|
@ -281,6 +282,7 @@ class Base:
|
|||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=())
|
||||
|
|
@ -291,6 +293,7 @@ class Base:
|
|||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
|
||||
])
|
||||
|
||||
lx = self.extractor_cls(attrs=None)
|
||||
|
|
|
|||
|
|
@ -117,12 +117,14 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
|
|||
def test_extraction(self):
|
||||
# Default arguments
|
||||
lx = HtmlParserLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(self.response),
|
||||
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
Link(url='http://example.com/innertag.html', text=u'inner tag'),
|
||||
Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
|
||||
])
|
||||
|
||||
def test_link_wrong_href(self):
|
||||
html = """
|
||||
|
|
@ -220,3 +222,9 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
|
|||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://b.com/test.html', text=u'', nofollow=False),
|
||||
])
|
||||
|
||||
@unittest.expectedFailure
|
||||
def test_extraction(self):
|
||||
# RegexLinkExtractor doesn't parse URLs with leading/trailing
|
||||
# whitespaces correctly.
|
||||
super(RegexLinkExtractorTestCase, self).test_extraction()
|
||||
|
|
|
|||
|
|
@ -36,6 +36,14 @@ class LoggingContribTest(unittest.TestCase):
|
|||
self.assertEqual(logline,
|
||||
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
|
||||
|
||||
def test_flags_in_request(self):
|
||||
req = Request("http://www.example.com", flags=['test','flag'])
|
||||
res = Response("http://www.example.com")
|
||||
logkws = self.formatter.crawled(req, res, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
self.assertEqual(logline,
|
||||
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
|
||||
|
||||
def test_dropped(self):
|
||||
item = {}
|
||||
exception = Exception(u"\u2018")
|
||||
|
|
|
|||
|
|
@ -211,9 +211,15 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
'TEST_ENABLED1': '1',
|
||||
'TEST_ENABLED2': True,
|
||||
'TEST_ENABLED3': 1,
|
||||
'TEST_ENABLED4': 'True',
|
||||
'TEST_ENABLED5': 'true',
|
||||
'TEST_ENABLED_WRONG': 'on',
|
||||
'TEST_DISABLED1': '0',
|
||||
'TEST_DISABLED2': False,
|
||||
'TEST_DISABLED3': 0,
|
||||
'TEST_DISABLED4': 'False',
|
||||
'TEST_DISABLED5': 'false',
|
||||
'TEST_DISABLED_WRONG': 'off',
|
||||
'TEST_INT1': 123,
|
||||
'TEST_INT2': '123',
|
||||
'TEST_FLOAT1': 123.45,
|
||||
|
|
@ -231,11 +237,15 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
self.assertTrue(settings.getbool('TEST_ENABLED1'))
|
||||
self.assertTrue(settings.getbool('TEST_ENABLED2'))
|
||||
self.assertTrue(settings.getbool('TEST_ENABLED3'))
|
||||
self.assertTrue(settings.getbool('TEST_ENABLED4'))
|
||||
self.assertTrue(settings.getbool('TEST_ENABLED5'))
|
||||
self.assertFalse(settings.getbool('TEST_ENABLEDx'))
|
||||
self.assertTrue(settings.getbool('TEST_ENABLEDx', True))
|
||||
self.assertFalse(settings.getbool('TEST_DISABLED1'))
|
||||
self.assertFalse(settings.getbool('TEST_DISABLED2'))
|
||||
self.assertFalse(settings.getbool('TEST_DISABLED3'))
|
||||
self.assertFalse(settings.getbool('TEST_DISABLED4'))
|
||||
self.assertFalse(settings.getbool('TEST_DISABLED5'))
|
||||
self.assertEqual(settings.getint('TEST_INT1'), 123)
|
||||
self.assertEqual(settings.getint('TEST_INT2'), 123)
|
||||
self.assertEqual(settings.getint('TEST_INTx'), 0)
|
||||
|
|
@ -258,6 +268,8 @@ class BaseSettingsTest(unittest.TestCase):
|
|||
self.assertEqual(settings.getdict('TEST_DICT3'), {})
|
||||
self.assertEqual(settings.getdict('TEST_DICT3', {'key1': 5}), {'key1': 5})
|
||||
self.assertRaises(ValueError, settings.getdict, 'TEST_LIST1')
|
||||
self.assertRaises(ValueError, settings.getbool, 'TEST_ENABLED_WRONG')
|
||||
self.assertRaises(ValueError, settings.getbool, 'TEST_DISABLED_WRONG')
|
||||
|
||||
def test_getpriority(self):
|
||||
settings = BaseSettings({'key': 'value'}, priority=99)
|
||||
|
|
|
|||
|
|
@ -345,7 +345,7 @@ Sitemap: /sitemap-relative-url.xml
|
|||
'http://www.example.com/sitemap-relative-url.xml'])
|
||||
|
||||
|
||||
class BaseSpiderDeprecationTest(unittest.TestCase):
|
||||
class DeprecationTest(unittest.TestCase):
|
||||
|
||||
def test_basespider_is_deprecated(self):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
|
|
@ -399,6 +399,29 @@ class BaseSpiderDeprecationTest(unittest.TestCase):
|
|||
assert isinstance(CrawlSpider(name='foo'), Spider)
|
||||
assert isinstance(CrawlSpider(name='foo'), BaseSpider)
|
||||
|
||||
def test_make_requests_from_url_deprecated(self):
|
||||
class MySpider4(Spider):
|
||||
name = 'spider1'
|
||||
start_urls = ['http://example.com']
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
class MySpider5(Spider):
|
||||
name = 'spider2'
|
||||
start_urls = ['http://example.com']
|
||||
|
||||
def make_requests_from_url(self, url):
|
||||
return Request(url + "/foo", dont_filter=True)
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
# spider without overridden make_requests_from_url method
|
||||
# doesn't issue a warning
|
||||
spider1 = MySpider4()
|
||||
self.assertEqual(len(list(spider1.start_requests())), 1)
|
||||
self.assertEqual(len(w), 0)
|
||||
|
||||
# spider with overridden make_requests_from_url issues a warning,
|
||||
# but the method still works
|
||||
spider2 = MySpider5()
|
||||
requests = list(spider2.start_requests())
|
||||
self.assertEqual(len(requests), 1)
|
||||
self.assertEqual(requests[0].url, 'http://example.com/foo')
|
||||
self.assertEqual(len(w), 1)
|
||||
|
|
|
|||
|
|
@ -60,7 +60,8 @@ def _responses(request, status_codes):
|
|||
class TestHttpErrorMiddleware(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.spider = Spider('foo')
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = Spider.from_crawler(crawler, name='foo')
|
||||
self.mw = HttpErrorMiddleware(Settings({}))
|
||||
self.req = Request('http://scrapytest.org')
|
||||
self.res200, self.res404 = _responses(self.req, [200, 404])
|
||||
|
|
@ -73,10 +74,10 @@ class TestHttpErrorMiddleware(TestCase):
|
|||
|
||||
def test_process_spider_exception(self):
|
||||
self.assertEquals([],
|
||||
self.mw.process_spider_exception(self.res404, \
|
||||
self.mw.process_spider_exception(self.res404,
|
||||
HttpError(self.res404), self.spider))
|
||||
self.assertEquals(None,
|
||||
self.mw.process_spider_exception(self.res404, \
|
||||
self.mw.process_spider_exception(self.res404,
|
||||
Exception(), self.spider))
|
||||
|
||||
def test_handle_httpstatus_list(self):
|
||||
|
|
@ -173,6 +174,12 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
|
|||
self.assertEqual(crawler.spider.parsed, {'200'})
|
||||
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
|
||||
|
||||
get_value = crawler.stats.get_value
|
||||
self.assertEqual(get_value('httperror/response_ignored_count'), 3)
|
||||
self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)
|
||||
self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)
|
||||
self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_logging(self):
|
||||
crawler = get_crawler(_HttpErrorSpider)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
from datetime import datetime
|
||||
import shutil
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.extensions.spiderstate import SpiderState
|
||||
|
|
@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase):
|
|||
def test_store_load(self):
|
||||
jobdir = self.mktemp()
|
||||
os.mkdir(jobdir)
|
||||
spider = Spider(name='default')
|
||||
dt = datetime.now()
|
||||
try:
|
||||
spider = Spider(name='default')
|
||||
dt = datetime.now()
|
||||
|
||||
ss = SpiderState(jobdir)
|
||||
ss.spider_opened(spider)
|
||||
spider.state['one'] = 1
|
||||
spider.state['dt'] = dt
|
||||
ss.spider_closed(spider)
|
||||
ss = SpiderState(jobdir)
|
||||
ss.spider_opened(spider)
|
||||
spider.state['one'] = 1
|
||||
spider.state['dt'] = dt
|
||||
ss.spider_closed(spider)
|
||||
|
||||
spider2 = Spider(name='default')
|
||||
ss2 = SpiderState(jobdir)
|
||||
ss2.spider_opened(spider2)
|
||||
self.assertEqual(spider.state, {'one': 1, 'dt': dt})
|
||||
ss2.spider_closed(spider2)
|
||||
spider2 = Spider(name='default')
|
||||
ss2 = SpiderState(jobdir)
|
||||
ss2.spider_opened(spider2)
|
||||
self.assertEqual(spider.state, {'one': 1, 'dt': dt})
|
||||
ss2.spider_closed(spider2)
|
||||
finally:
|
||||
shutil.rmtree(jobdir)
|
||||
|
||||
def test_state_attribute(self):
|
||||
# state attribute must be present if jobdir is not set, to provide a
|
||||
|
|
|
|||
|
|
@ -62,6 +62,27 @@ class BuildComponentListTest(unittest.TestCase):
|
|||
self.assertRaises(ValueError, build_component_list, duplicate_bs,
|
||||
convert=lambda x: x.lower())
|
||||
|
||||
def test_valid_numbers(self):
|
||||
# work well with None and numeric values
|
||||
d = {'a': 10, 'b': None, 'c': 15, 'd': 5.0}
|
||||
self.assertEqual(build_component_list(d, convert=lambda x: x),
|
||||
['d', 'a', 'c'])
|
||||
d = {'a': 33333333333333333333, 'b': 11111111111111111111, 'c': 22222222222222222222}
|
||||
self.assertEqual(build_component_list(d, convert=lambda x: x),
|
||||
['b', 'c', 'a'])
|
||||
# raise exception for invalid values
|
||||
d = {'one': '5'}
|
||||
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
|
||||
d = {'one': '1.0'}
|
||||
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
|
||||
d = {'one': [1, 2, 3]}
|
||||
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
|
||||
d = {'one': {'a': 'a', 'b': 2}}
|
||||
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
|
||||
d = {'one': 'lorem ipsum',}
|
||||
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
|
||||
|
||||
|
||||
|
||||
class UtilsConfTestCase(unittest.TestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -31,5 +31,8 @@ class ProjectUtilsTest(unittest.TestCase):
|
|||
def test_data_path_inside_project(self):
|
||||
with inside_a_project() as proj_path:
|
||||
expected = os.path.join(proj_path, '.scrapy', 'somepath')
|
||||
self.assertEquals(expected, data_path('somepath'))
|
||||
self.assertEquals(
|
||||
os.path.realpath(expected),
|
||||
os.path.realpath(data_path('somepath'))
|
||||
)
|
||||
self.assertEquals('/absolute/path', data_path('/absolute/path'))
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import unittest
|
||||
|
||||
from scrapy.http import Request
|
||||
from scrapy.http import Request, FormRequest
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.reqser import request_to_dict, request_from_dict
|
||||
|
||||
|
|
@ -25,7 +25,8 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
cookies={'currency': u'руб'},
|
||||
encoding='latin-1',
|
||||
priority=20,
|
||||
meta={'a': 'b'})
|
||||
meta={'a': 'b'},
|
||||
flags=['testFlag'])
|
||||
self._assert_serializes_ok(r)
|
||||
|
||||
def test_latin1_body(self):
|
||||
|
|
@ -42,6 +43,7 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
self._assert_same_request(request, request2)
|
||||
|
||||
def _assert_same_request(self, r1, r2):
|
||||
self.assertEqual(r1.__class__, r2.__class__)
|
||||
self.assertEqual(r1.url, r2.url)
|
||||
self.assertEqual(r1.callback, r2.callback)
|
||||
self.assertEqual(r1.errback, r2.errback)
|
||||
|
|
@ -53,6 +55,13 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
self.assertEqual(r1._encoding, r2._encoding)
|
||||
self.assertEqual(r1.priority, r2.priority)
|
||||
self.assertEqual(r1.dont_filter, r2.dont_filter)
|
||||
self.assertEqual(r1.flags, r2.flags)
|
||||
|
||||
def test_request_class(self):
|
||||
r = FormRequest("http://www.example.com")
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
r = CustomRequest("http://www.example.com")
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
||||
def test_callback_serialization(self):
|
||||
r = Request("http://www.example.com", callback=self.spider.parse_item,
|
||||
|
|
@ -77,3 +86,7 @@ class TestSpider(Spider):
|
|||
|
||||
def handle_error(self, failure):
|
||||
pass
|
||||
|
||||
|
||||
class CustomRequest(Request):
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ Tests borrowed from the twisted.web.client tests.
|
|||
"""
|
||||
import os
|
||||
import six
|
||||
from six.moves.urllib.parse import urlparse
|
||||
import shutil
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.web import server, static, util, resource
|
||||
|
|
@ -12,6 +12,7 @@ from twisted.internet import reactor, defer
|
|||
from twisted.test.proto_helpers import StringTransport
|
||||
from twisted.python.filepath import FilePath
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.internet.defer import inlineCallbacks
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.http import Request, Headers
|
||||
|
|
@ -229,10 +230,10 @@ class WebClientTestCase(unittest.TestCase):
|
|||
return reactor.listenTCP(0, site, interface="127.0.0.1")
|
||||
|
||||
def setUp(self):
|
||||
name = self.mktemp()
|
||||
os.mkdir(name)
|
||||
FilePath(name).child("file").setContent(b"0123456789")
|
||||
r = static.File(name)
|
||||
self.tmpname = self.mktemp()
|
||||
os.mkdir(self.tmpname)
|
||||
FilePath(self.tmpname).child("file").setContent(b"0123456789")
|
||||
r = static.File(self.tmpname)
|
||||
r.putChild(b"redirect", util.Redirect(b"/file"))
|
||||
r.putChild(b"wait", ForeverTakingResource())
|
||||
r.putChild(b"error", ErrorResource())
|
||||
|
|
@ -246,8 +247,10 @@ class WebClientTestCase(unittest.TestCase):
|
|||
self.port = self._listen(self.wrapper)
|
||||
self.portno = self.port.getHost().port
|
||||
|
||||
@inlineCallbacks
|
||||
def tearDown(self):
|
||||
return self.port.stopListening()
|
||||
yield self.port.stopListening()
|
||||
shutil.rmtree(self.tmpname)
|
||||
|
||||
def getURL(self, path):
|
||||
return "http://127.0.0.1:%d/%s" % (self.portno, path)
|
||||
|
|
@ -266,7 +269,6 @@ class WebClientTestCase(unittest.TestCase):
|
|||
getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
|
||||
self.assertEquals, to_bytes("www.example.com"))])
|
||||
|
||||
|
||||
def test_getPage(self):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
|
|
@ -276,7 +278,6 @@ class WebClientTestCase(unittest.TestCase):
|
|||
d.addCallback(self.assertEquals, b"0123456789")
|
||||
return d
|
||||
|
||||
|
||||
def test_getPageHead(self):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
|
|
@ -289,7 +290,6 @@ class WebClientTestCase(unittest.TestCase):
|
|||
_getPage("head").addCallback(self.assertEqual, b""),
|
||||
_getPage("HEAD").addCallback(self.assertEqual, b"")])
|
||||
|
||||
|
||||
def test_timeoutNotTriggering(self):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and the page is
|
||||
|
|
@ -301,7 +301,6 @@ class WebClientTestCase(unittest.TestCase):
|
|||
self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
|
||||
return d
|
||||
|
||||
|
||||
def test_timeoutTriggering(self):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and that many
|
||||
|
|
@ -351,7 +350,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
b' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
|
||||
b'<a href="/file">click here</a>\n </body>\n</html>\n')
|
||||
|
||||
def test_Encoding(self):
|
||||
def test_encoding(self):
|
||||
""" Test that non-standart body encoding matches
|
||||
Content-Encoding header """
|
||||
body = b'\xd0\x81\xd1\x8e\xd0\xaf'
|
||||
|
|
|
|||
21
tox.ini
21
tox.ini
|
|
@ -21,16 +21,16 @@ passenv =
|
|||
commands =
|
||||
py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
|
||||
|
||||
[testenv:precise]
|
||||
[testenv:trusty]
|
||||
basepython = python2.7
|
||||
deps =
|
||||
pyOpenSSL==0.13
|
||||
lxml==2.3.2
|
||||
Twisted==11.1.0
|
||||
boto==2.2.2
|
||||
Pillow<2.0
|
||||
lxml==3.3.3
|
||||
Twisted==13.2.0
|
||||
boto==2.20.1
|
||||
Pillow==2.3.0
|
||||
cssselect==0.9.1
|
||||
zope.interface==3.6.1
|
||||
zope.interface==4.0.5
|
||||
-rtests/requirements.txt
|
||||
|
||||
[testenv:jessie]
|
||||
|
|
@ -54,6 +54,11 @@ commands =
|
|||
pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
|
||||
py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
|
||||
|
||||
[testenv:pypy]
|
||||
basepython = pypy
|
||||
commands =
|
||||
py.test {posargs:scrapy tests}
|
||||
|
||||
[testenv:py33]
|
||||
basepython = python3.3
|
||||
deps =
|
||||
|
|
@ -70,6 +75,10 @@ deps = {[testenv:py33]deps}
|
|||
basepython = python3.5
|
||||
deps = {[testenv:py33]deps}
|
||||
|
||||
[testenv:py36]
|
||||
basepython = python3.6
|
||||
deps = {[testenv:py33]deps}
|
||||
|
||||
[docs]
|
||||
changedir = docs
|
||||
deps =
|
||||
|
|
|
|||
Loading…
Reference in New Issue