mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into pylint
This commit is contained in:
Adrián Chaves
GitHub
commit
40ed184914
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 1.7.0
|
||||
current_version = 1.8.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
27
.travis.yml
27
.travis.yml
|
|
@ -7,16 +7,10 @@ branches:
|
|||
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
|
||||
matrix:
|
||||
include:
|
||||
- env: TOXENV=flake8
|
||||
python: 3.8
|
||||
- env: TOXENV=pylint
|
||||
python: 3.7
|
||||
- env: TOXENV=py27
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-pinned
|
||||
python: 2.7
|
||||
- env: TOXENV=py27-extra-deps
|
||||
python: 2.7
|
||||
- env: TOXENV=pypy
|
||||
python: 2.7
|
||||
python: 3.8
|
||||
- env: TOXENV=pypy3
|
||||
python: 3.5
|
||||
- env: TOXENV=py35
|
||||
|
|
@ -27,19 +21,14 @@ matrix:
|
|||
python: 3.6
|
||||
- env: TOXENV=py37
|
||||
python: 3.7
|
||||
- env: TOXENV=py37-extra-deps
|
||||
python: 3.7
|
||||
- env: TOXENV=py38
|
||||
python: 3.8
|
||||
- env: TOXENV=py38-extra-deps
|
||||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.6
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy" ]; then
|
||||
export PYPY_VERSION="pypy-6.0.0-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
fi
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
|
|
@ -70,4 +59,4 @@ deploy:
|
|||
on:
|
||||
tags: true
|
||||
repo: scrapy/scrapy
|
||||
condition: "$TOXENV == py27 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ https://scrapy.org
|
|||
Requirements
|
||||
============
|
||||
|
||||
* Python 2.7 or Python 3.5+
|
||||
* Python 3.5+
|
||||
* Works on Linux, Windows, Mac OSX, BSD
|
||||
|
||||
Install
|
||||
|
|
|
|||
10
conftest.py
10
conftest.py
|
|
@ -19,3 +19,13 @@ if six.PY3:
|
|||
def chdir(tmpdir):
|
||||
"""Change to pytest-provided temporary directory"""
|
||||
tmpdir.chdir()
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(session, config, items):
|
||||
# Avoid executing tests when executing `--flake8` flag (pytest-flake8)
|
||||
try:
|
||||
from pytest_flake8 import Flake8Item
|
||||
if config.getoption('--flake8'):
|
||||
items[:] = [item for item in items if isinstance(item, Flake8Item)]
|
||||
except ImportError:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -69,11 +69,11 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 2.7 and Python 3.5+
|
||||
Scrapy is supported under Python 3.5+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 2.6 support was dropped starting at Scrapy 0.20.
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
Python 2 support was dropped in Scrapy 2.0.
|
||||
|
||||
.. note::
|
||||
For Python 3 support on Windows, it is recommended to use
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Installation guide
|
|||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 2.7 and Python 3.5 or above
|
||||
Scrapy runs on Python 3.5 or above
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
|
|
@ -102,10 +102,8 @@ just like any other Python package.
|
|||
(See :ref:`platform-specific guides <intro-install-platform-notes>`
|
||||
below for non-Python dependencies that you may need to install beforehand).
|
||||
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default.
|
||||
|
||||
* If you want to install scrapy with Python 3, install scrapy within a Python 3 virtualenv.
|
||||
* And if you want to install scrapy with Python 2, install scrapy within a Python 2 virtualenv.
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
|
||||
only supports Python 3, make sure you created a Python 3 virtualenv.
|
||||
|
||||
.. _virtualenv: https://virtualenv.pypa.io
|
||||
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
|
||||
|
|
@ -149,16 +147,12 @@ typically too old and slow to catch up with latest Scrapy.
|
|||
To install scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
|
||||
these dependencies::
|
||||
|
||||
sudo apt-get install python-dev python-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
sudo apt-get install python3 python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev libffi-dev libssl-dev
|
||||
|
||||
- ``python-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
|
||||
- ``python3-dev``, ``zlib1g-dev``, ``libxml2-dev`` and ``libxslt1-dev``
|
||||
are required for ``lxml``
|
||||
- ``libssl-dev`` and ``libffi-dev`` are required for ``cryptography``
|
||||
|
||||
If you want to install scrapy on Python 3, you’ll also need Python 3 development headers::
|
||||
|
||||
sudo apt-get install python3 python3-dev
|
||||
|
||||
Inside a :ref:`virtualenv <intro-using-virtualenv>`,
|
||||
you can install Scrapy with ``pip`` after that::
|
||||
|
||||
|
|
|
|||
238
docs/news.rst
238
docs/news.rst
|
|
@ -6,22 +6,246 @@ Release notes
|
|||
.. note:: Scrapy 1.x will be the last series supporting Python 2. Scrapy 2.0,
|
||||
planned for Q4 2019 or Q1 2020, will support **Python 3 only**.
|
||||
|
||||
.. _release-1.8.0:
|
||||
|
||||
Scrapy 1.8.0 (2019-10-28)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* Dropped Python 3.4 support and updated minimum requirements; made Python 3.8
|
||||
support official
|
||||
* New :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class method
|
||||
* New :setting:`ROBOTSTXT_PARSER` and :setting:`ROBOTSTXT_USER_AGENT` settings
|
||||
* New :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
|
||||
:setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` settings
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Python 3.4 is no longer supported, and some of the minimum requirements of
|
||||
Scrapy have also changed:
|
||||
|
||||
* cssselect_ 0.9.1
|
||||
* cryptography_ 2.0
|
||||
* lxml_ 3.5.0
|
||||
* pyOpenSSL_ 16.2.0
|
||||
* queuelib_ 1.4.2
|
||||
* service_identity_ 16.0.0
|
||||
* six_ 1.10.0
|
||||
* Twisted_ 17.9.0 (16.0.0 with Python 2)
|
||||
* zope.interface_ 4.1.3
|
||||
|
||||
(:issue:`3892`)
|
||||
|
||||
* ``JSONRequest`` is now called :class:`~scrapy.http.JsonRequest` for
|
||||
consistency with similar classes (:issue:`3929`, :issue:`3982`)
|
||||
|
||||
* If you are using a custom context factory
|
||||
(:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
|
||||
accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
|
||||
(:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
|
||||
|
||||
* :class:`~scrapy.loader.ItemLoader` now turns the values of its input item
|
||||
into lists::
|
||||
|
||||
>>> item = MyItem()
|
||||
>>> item['field'] = 'value1'
|
||||
>>> loader = ItemLoader(item=item)
|
||||
>>> item['field']
|
||||
['value1']
|
||||
|
||||
This is needed to allow adding values to existing fields
|
||||
(``loader.add_value('field', 'value2')``).
|
||||
|
||||
(:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
|
||||
:issue:`4036`)
|
||||
|
||||
See also :ref:`1.8-deprecation-removals` below.
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* A new :meth:`Request.from_curl <scrapy.http.Request.from_curl>` class
|
||||
method allows :ref:`creating a request from a cURL command
|
||||
<requests-from-curl>` (:issue:`2985`, :issue:`3862`)
|
||||
|
||||
* A new :setting:`ROBOTSTXT_PARSER` setting allows choosing which robots.txt_
|
||||
parser to use. It includes built-in support for
|
||||
:ref:`RobotFileParser <python-robotfileparser>`,
|
||||
:ref:`Protego <protego-parser>` (default), :ref:`Reppy <reppy-parser>`, and
|
||||
:ref:`Robotexclusionrulesparser <rerp-parser>`, and allows you to
|
||||
:ref:`implement support for additional parsers
|
||||
<support-for-new-robots-parser>` (:issue:`754`, :issue:`2669`,
|
||||
:issue:`3796`, :issue:`3935`, :issue:`3969`, :issue:`4006`)
|
||||
|
||||
* A new :setting:`ROBOTSTXT_USER_AGENT` setting allows defining a separate
|
||||
user agent string to use for robots.txt_ parsing (:issue:`3931`,
|
||||
:issue:`3966`)
|
||||
|
||||
* :class:`~scrapy.spiders.Rule` no longer requires a :class:`LinkExtractor
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` parameter
|
||||
(:issue:`781`, :issue:`4016`)
|
||||
|
||||
* Use the new :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting to customize
|
||||
the TLS/SSL ciphers used by the default HTTP/1.1 downloader (:issue:`3392`,
|
||||
:issue:`3442`)
|
||||
|
||||
* Set the new :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING` setting to
|
||||
``True`` to enable debug-level messages about TLS connection parameters
|
||||
after establishing HTTPS connections (:issue:`2111`, :issue:`3450`)
|
||||
|
||||
* Callbacks that receive keyword arguments
|
||||
(see :attr:`Request.cb_kwargs <scrapy.http.Request.cb_kwargs>`) can now be
|
||||
tested using the new :class:`@cb_kwargs
|
||||
<scrapy.contracts.default.CallbackKeywordArgumentsContract>`
|
||||
:ref:`spider contract <topics-contracts>` (:issue:`3985`, :issue:`3988`)
|
||||
|
||||
* When a :class:`@scrapes <scrapy.contracts.default.ScrapesContract>` spider
|
||||
contract fails, all missing fields are now reported (:issue:`766`,
|
||||
:issue:`3939`)
|
||||
|
||||
* :ref:`Custom log formats <custom-log-formats>` can now drop messages by
|
||||
having the corresponding methods of the configured :setting:`LOG_FORMATTER`
|
||||
return ``None`` (:issue:`3984`, :issue:`3987`)
|
||||
|
||||
* A much improved completion definition is now available for Zsh_
|
||||
(:issue:`4069`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* :meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` no
|
||||
longer makes later calls to :meth:`ItemLoader.get_output_value()
|
||||
<scrapy.loader.ItemLoader.get_output_value>` or
|
||||
:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` return
|
||||
empty data (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`,
|
||||
:issue:`3998`, :issue:`4036`)
|
||||
|
||||
* Fixed :class:`~scrapy.statscollectors.DummyStatsCollector` raising a
|
||||
:exc:`TypeError` exception (:issue:`4007`, :issue:`4052`)
|
||||
|
||||
* :meth:`FilesPipeline.file_path
|
||||
<scrapy.pipelines.files.FilesPipeline.file_path>` and
|
||||
:meth:`ImagesPipeline.file_path
|
||||
<scrapy.pipelines.images.ImagesPipeline.file_path>` no longer choose
|
||||
file extensions that are not `registered with IANA`_ (:issue:`1287`,
|
||||
:issue:`3953`, :issue:`3954`)
|
||||
|
||||
* When using botocore_ to persist files in S3, all botocore-supported headers
|
||||
are properly mapped now (:issue:`3904`, :issue:`3905`)
|
||||
|
||||
* FTP passwords in :setting:`FEED_URI` containing percent-escaped characters
|
||||
are now properly decoded (:issue:`3941`)
|
||||
|
||||
* A memory-handling and error-handling issue in
|
||||
:func:`scrapy.utils.ssl.get_temp_key_info` has been fixed (:issue:`3920`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* The documentation now covers how to define and configure a :ref:`custom log
|
||||
format <custom-log-formats>` (:issue:`3616`, :issue:`3660`)
|
||||
|
||||
* API documentation added for :class:`~scrapy.exporters.MarshalItemExporter`
|
||||
and :class:`~scrapy.exporters.PythonItemExporter` (:issue:`3973`)
|
||||
|
||||
* API documentation added for :class:`~scrapy.item.BaseItem` and
|
||||
:class:`~scrapy.item.ItemMeta` (:issue:`3999`)
|
||||
|
||||
* Minor documentation fixes (:issue:`2998`, :issue:`3398`, :issue:`3597`,
|
||||
:issue:`3894`, :issue:`3934`, :issue:`3978`, :issue:`3993`, :issue:`4022`,
|
||||
:issue:`4028`, :issue:`4033`, :issue:`4046`, :issue:`4050`, :issue:`4055`,
|
||||
:issue:`4056`, :issue:`4061`, :issue:`4072`, :issue:`4071`, :issue:`4079`,
|
||||
:issue:`4081`, :issue:`4089`, :issue:`4093`)
|
||||
|
||||
|
||||
.. _1.8-deprecation-removals:
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* ``scrapy.xlib`` has been removed (:issue:`4015`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* The LevelDB_ storage backend
|
||||
(``scrapy.extensions.httpcache.LeveldbCacheStorage``) of
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` is
|
||||
deprecated (:issue:`4085`, :issue:`4092`)
|
||||
|
||||
* Use of the undocumented ``SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE`` environment
|
||||
variable is deprecated (:issue:`3910`)
|
||||
|
||||
* ``scrapy.item.DictItem`` is deprecated, use :class:`~scrapy.item.Item`
|
||||
instead (:issue:`3999`)
|
||||
|
||||
|
||||
Other changes
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Minimum versions of optional Scrapy requirements that are covered by
|
||||
continuous integration tests have been updated:
|
||||
|
||||
* botocore_ 1.3.23
|
||||
* Pillow_ 3.4.2
|
||||
|
||||
Lower versions of these optional requirements may work, but it is not
|
||||
guaranteed (:issue:`3892`)
|
||||
|
||||
* GitHub templates for bug reports and feature requests (:issue:`3126`,
|
||||
:issue:`3471`, :issue:`3749`, :issue:`3754`)
|
||||
|
||||
* Continuous integration fixes (:issue:`3923`)
|
||||
|
||||
* Code cleanup (:issue:`3391`, :issue:`3907`, :issue:`3946`, :issue:`3950`,
|
||||
:issue:`4023`, :issue:`4031`)
|
||||
|
||||
|
||||
.. _release-1.7.4:
|
||||
|
||||
Scrapy 1.7.4 (2019-10-21)
|
||||
-------------------------
|
||||
|
||||
Revert the fix for :issue:`3804` (:issue:`3819`), which has a few undesired
|
||||
side effects (:issue:`3897`, :issue:`3976`).
|
||||
|
||||
As a result, when an item loader is initialized with an item,
|
||||
:meth:`ItemLoader.load_item() <scrapy.loader.ItemLoader.load_item>` once again
|
||||
makes later calls to :meth:`ItemLoader.get_output_value()
|
||||
<scrapy.loader.ItemLoader.get_output_value>` or :meth:`ItemLoader.load_item()
|
||||
<scrapy.loader.ItemLoader.load_item>` return empty data.
|
||||
|
||||
|
||||
.. _release-1.7.3:
|
||||
|
||||
Scrapy 1.7.3 (2019-08-01)
|
||||
-------------------------
|
||||
|
||||
Enforce lxml 4.3.5 or lower for Python 3.4 (:issue:`3912`, :issue:`3918`).
|
||||
|
||||
|
||||
.. _release-1.7.2:
|
||||
|
||||
Scrapy 1.7.2 (2019-07-23)
|
||||
-------------------------
|
||||
|
||||
Fix Python 2 support (:issue:`3889`, :issue:`3893`, :issue:`3896`).
|
||||
|
||||
|
||||
.. _release-1.7.1:
|
||||
|
||||
Scrapy 1.7.1 (2019-07-18)
|
||||
-------------------------
|
||||
|
||||
Re-packaging of Scrapy 1.7.0, which was missing some changes in PyPI.
|
||||
|
||||
|
||||
.. _release-1.7.0:
|
||||
|
||||
Scrapy 1.7.0 (2019-07-18)
|
||||
|
|
@ -556,7 +780,7 @@ Scrapy 1.5.2 (2019-01-22)
|
|||
|
||||
See :ref:`telnet console <topics-telnetconsole>` documentation for more info
|
||||
|
||||
* Backport CI build failure under GCE environemnt due to boto import error.
|
||||
* Backport CI build failure under GCE environment due to boto import error.
|
||||
|
||||
.. _release-1.5.1:
|
||||
|
||||
|
|
@ -2818,23 +3042,35 @@ First release of Scrapy.
|
|||
|
||||
|
||||
.. _AJAX crawleable urls: https://developers.google.com/webmasters/ajax-crawling/docs/getting-started?csw=1
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
.. _ClientForm: http://wwwsearch.sourceforge.net/old/ClientForm/
|
||||
.. _Creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
|
||||
.. _cryptography: https://cryptography.io/en/latest/
|
||||
.. _cssselect: https://github.com/scrapy/cssselect/
|
||||
.. _docstrings: https://docs.python.org/glossary.html#term-docstring
|
||||
.. _KeyboardInterrupt: https://docs.python.org/library/exceptions.html#KeyboardInterrupt
|
||||
.. _LevelDB: https://github.com/google/leveldb
|
||||
.. _lxml: http://lxml.de/
|
||||
.. _marshal: https://docs.python.org/2/library/marshal.html
|
||||
.. _parsel.csstranslator.GenericTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.GenericTranslator
|
||||
.. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator
|
||||
.. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr
|
||||
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
|
||||
.. _Pillow: https://python-pillow.org/
|
||||
.. _pyOpenSSL: https://www.pyopenssl.org/en/stable/
|
||||
.. _queuelib: https://github.com/scrapy/queuelib
|
||||
.. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml
|
||||
.. _resource: https://docs.python.org/2/library/resource.html
|
||||
.. _robots.txt: http://www.robotstxt.org/
|
||||
.. _scrapely: https://github.com/scrapy/scrapely
|
||||
.. _service_identity: https://service-identity.readthedocs.io/en/stable/
|
||||
.. _six: https://six.readthedocs.io/
|
||||
.. _tox: https://pypi.python.org/pypi/tox
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _w3lib: https://github.com/scrapy/w3lib
|
||||
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
|
||||
.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
|
||||
.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
|
||||
.. _Zsh: https://www.zsh.org/
|
||||
|
|
|
|||
|
|
@ -348,7 +348,6 @@ HttpCacheMiddleware
|
|||
|
||||
* :ref:`httpcache-storage-fs`
|
||||
* :ref:`httpcache-storage-dbm`
|
||||
* :ref:`httpcache-storage-leveldb`
|
||||
|
||||
You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE`
|
||||
setting. Or you can also :ref:`implement your own storage backend. <httpcache-storage-custom>`
|
||||
|
|
@ -478,27 +477,6 @@ DBM storage backend
|
|||
By default, it uses the anydbm_ module, but you can change it with the
|
||||
:setting:`HTTPCACHE_DBM_MODULE` setting.
|
||||
|
||||
.. _httpcache-storage-leveldb:
|
||||
|
||||
LevelDB storage backend
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
.. class:: LeveldbCacheStorage
|
||||
|
||||
.. versionadded:: 0.23
|
||||
|
||||
A LevelDB_ storage backend is also available for the HTTP cache middleware.
|
||||
|
||||
This backend is not recommended for development because only one process
|
||||
can access LevelDB databases at the same time, so you can't run a crawl and
|
||||
open the scrapy shell in parallel for the same spider.
|
||||
|
||||
In order to use this storage backend, install the `LevelDB python
|
||||
bindings`_ (e.g. ``pip install leveldb``).
|
||||
|
||||
.. _LevelDB: https://github.com/google/leveldb
|
||||
.. _leveldb python bindings: https://pypi.python.org/pypi/leveldb
|
||||
|
||||
.. _httpcache-storage-custom:
|
||||
|
||||
Writing your own storage backend
|
||||
|
|
|
|||
|
|
@ -99,12 +99,12 @@ The storages backends supported out of the box are:
|
|||
|
||||
* :ref:`topics-feed-storage-fs`
|
||||
* :ref:`topics-feed-storage-ftp`
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_ or boto_)
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
* :ref:`topics-feed-storage-stdout`
|
||||
|
||||
Some storage backends may be unavailable if the required external libraries are
|
||||
not available. For example, the S3 backend is only available if the botocore_
|
||||
or boto_ library is installed (Scrapy supports boto_ only on Python 2).
|
||||
library is installed.
|
||||
|
||||
|
||||
.. _topics-feed-uri-params:
|
||||
|
|
@ -182,7 +182,7 @@ The feeds are stored on `Amazon S3`_.
|
|||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `botocore`_ (Python 2 and Python 3) or `boto`_ (Python 2 only)
|
||||
* Required external libraries: `botocore`_
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
|
@ -399,6 +399,5 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
|
||||
.. _URI: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _boto: https://github.com/boto/boto
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ knowledge about Python internals. For more info about Guppy, refer to the
|
|||
|
||||
Debugging memory leaks with muppy
|
||||
=================================
|
||||
If you're using Python 3, you can use muppy from `Pympler`_.
|
||||
You can use muppy from `Pympler`_.
|
||||
|
||||
.. _Pympler: https://pypi.org/project/Pympler/
|
||||
|
||||
|
|
|
|||
|
|
@ -35,6 +35,12 @@ Then, you start collecting values into the Item Loader, typically using
|
|||
the same item field; the Item Loader will know how to "join" those values later
|
||||
using a proper processing function.
|
||||
|
||||
.. note:: Collected data is internally stored as lists,
|
||||
allowing to add several values to the same field.
|
||||
If an ``item`` argument is passed when creating a loader,
|
||||
each of the item's values will be stored as-is if it's already
|
||||
an iterable, or wrapped with a list if it's a single value.
|
||||
|
||||
Here is a typical Item Loader usage in a :ref:`Spider <topics-spiders>`, using
|
||||
the :ref:`Product item <topics-items-declaring>` declared in the :ref:`Items
|
||||
chapter <topics-items>`::
|
||||
|
|
@ -128,9 +134,9 @@ So what happens is:
|
|||
It's worth noticing that processors are just callable objects, which are called
|
||||
with the data to be parsed, and return a parsed value. So you can use any
|
||||
function as input or output processor. The only requirement is that they must
|
||||
accept one (and only one) positional argument, which will be an iterator.
|
||||
accept one (and only one) positional argument, which will be an iterable.
|
||||
|
||||
.. note:: Both input and output processors must receive an iterator as their
|
||||
.. note:: Both input and output processors must receive an iterable as their
|
||||
first argument. The output of those functions can be anything. The result of
|
||||
input processors will be appended to an internal list (in the Loader)
|
||||
containing the collected values (for that field). The result of the output
|
||||
|
|
|
|||
|
|
@ -198,8 +198,9 @@ to override some of the Scrapy settings regarding logging.
|
|||
Custom Log Formats
|
||||
------------------
|
||||
|
||||
A custom log format can be set for different actions by extending :class:`~scrapy.logformatter.LogFormatter` class
|
||||
and making :setting:`LOG_FORMATTER` point to your new class.
|
||||
A custom log format can be set for different actions by extending
|
||||
:class:`~scrapy.logformatter.LogFormatter` class and making
|
||||
:setting:`LOG_FORMATTER` point to your new class.
|
||||
|
||||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ policy::
|
|||
|
||||
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
|
||||
|
||||
Because Scrapy uses ``boto`` / ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
|
||||
|
||||
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
|
||||
|
|
|
|||
|
|
@ -596,8 +596,8 @@ Response objects
|
|||
(for single valued headers) or lists (for multi-valued headers).
|
||||
:type headers: dict
|
||||
|
||||
:param body: the response body. To access the decoded text as str (unicode
|
||||
in Python 2) you can use ``response.text`` from an encoding-aware
|
||||
:param body: the response body. To access the decoded text as str you can use
|
||||
``response.text`` from an encoding-aware
|
||||
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
|
||||
such as :class:`TextResponse`.
|
||||
:type body: bytes
|
||||
|
|
|
|||
|
|
@ -188,7 +188,6 @@ AWS_ENDPOINT_URL
|
|||
Default: ``None``
|
||||
|
||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: AWS_USE_SSL
|
||||
|
||||
|
|
@ -199,7 +198,6 @@ Default: ``None``
|
|||
|
||||
Use this option if you want to disable SSL connection for communication with
|
||||
S3 or S3-like storage. By default SSL will be used.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: AWS_VERIFY
|
||||
|
||||
|
|
@ -209,7 +207,7 @@ AWS_VERIFY
|
|||
Default: ``None``
|
||||
|
||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||
SSL verification will occur. Only supported with ``botocore`` library.
|
||||
SSL verification will occur.
|
||||
|
||||
.. setting:: AWS_REGION_NAME
|
||||
|
||||
|
|
@ -219,7 +217,6 @@ AWS_REGION_NAME
|
|||
Default: ``None``
|
||||
|
||||
The name of the region associated with the AWS client.
|
||||
Only supported with ``botocore`` library.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
|
|
|
|||
|
|
@ -72,8 +72,6 @@ scrapy.Spider
|
|||
spider that crawls ``mywebsite.com`` would often be called
|
||||
``mywebsite``.
|
||||
|
||||
.. note:: In Python 2 this must be ASCII only.
|
||||
|
||||
.. attribute:: allowed_domains
|
||||
|
||||
An optional list of strings containing domains that this spider is
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ _scrapy() {
|
|||
'-c[evaluate the code in the shell, print the result and exit]:code:(CODE)'
|
||||
'--no-redirect[do not handle HTTP 3xx status codes and print response as-is]'
|
||||
'--spider[use this spider]:spider:_scrapy_spiders'
|
||||
'::file:_files -g \*.http'
|
||||
'::file:_files -g \*.html'
|
||||
'::URL:_httpie_urls'
|
||||
)
|
||||
_scrapy_glb_opts $options
|
||||
|
|
|
|||
248
pytest.ini
248
pytest.ini
|
|
@ -4,3 +4,251 @@ python_files=test_*.py __init__.py
|
|||
python_classes=
|
||||
addopts = --doctest-modules --assert=plain
|
||||
twisted = 1
|
||||
flake8-ignore =
|
||||
# extras
|
||||
extras/qps-bench-server.py E261 E501
|
||||
extras/qpsclient.py E501 E261 E501
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128 E501
|
||||
scrapy/commands/check.py F401 E501
|
||||
scrapy/commands/crawl.py E501
|
||||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/list.py E302
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E302 E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
scrapy/commands/startproject.py E502 E127 E501 E128
|
||||
scrapy/commands/version.py E501 E128
|
||||
scrapy/commands/view.py F401 E302
|
||||
# scrapy/contracts
|
||||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E502 E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E261 E501 E128 E127 E306 E502
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E261 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226
|
||||
scrapy/core/downloader/__init__.py F401 E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/http.py F401
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
scrapy/downloadermiddlewares/httpcompression.py E502 E128
|
||||
scrapy/downloadermiddlewares/httpproxy.py E501
|
||||
scrapy/downloadermiddlewares/redirect.py E501 W504
|
||||
scrapy/downloadermiddlewares/retry.py E501 E126
|
||||
scrapy/downloadermiddlewares/robotstxt.py F401 E501
|
||||
scrapy/downloadermiddlewares/stats.py E501
|
||||
# scrapy/extensions
|
||||
scrapy/extensions/closespider.py E501 E502 E128 E123
|
||||
scrapy/extensions/corestats.py E302 E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303 F401
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E302 E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
scrapy/extensions/throttle.py E501
|
||||
# scrapy/http
|
||||
scrapy/http/__init__.py F401
|
||||
scrapy/http/common.py E501
|
||||
scrapy/http/cookies.py E501
|
||||
scrapy/http/request/__init__.py E501
|
||||
scrapy/http/request/form.py E501 E123
|
||||
scrapy/http/request/json_request.py E501
|
||||
scrapy/http/response/__init__.py E501 E128 W293 W291
|
||||
scrapy/http/response/html.py E302
|
||||
scrapy/http/response/text.py E501 W293 E128 E124
|
||||
scrapy/http/response/xml.py E302
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E502 E128
|
||||
scrapy/loader/common.py E302
|
||||
scrapy/loader/processors.py E501
|
||||
# scrapy/pipelines
|
||||
scrapy/pipelines/__init__.py E302
|
||||
scrapy/pipelines/files.py E116 E501 E266
|
||||
scrapy/pipelines/images.py E265 E501
|
||||
scrapy/pipelines/media.py E125 E501 E266
|
||||
# scrapy/selector
|
||||
scrapy/selector/__init__.py F403 F401
|
||||
scrapy/selector/unified.py F401 E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E261 E114 E116 E226
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
scrapy/spidermiddlewares/offsite.py E501
|
||||
scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
|
||||
scrapy/spidermiddlewares/urllength.py E501
|
||||
# scrapy/spiders
|
||||
scrapy/spiders/__init__.py F401 E501 E402
|
||||
scrapy/spiders/crawl.py E501
|
||||
scrapy/spiders/feed.py E501 E261
|
||||
scrapy/spiders/sitemap.py E501
|
||||
# scrapy/utils
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/boto.py F401
|
||||
scrapy/utils/conf.py E402 E502 E501
|
||||
scrapy/utils/console.py E302 E261 F401 E306 E305
|
||||
scrapy/utils/curl.py F401
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/decorators.py E501 E302
|
||||
scrapy/utils/defer.py E501 E302 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/display.py E302
|
||||
scrapy/utils/engine.py F401 E261 E302
|
||||
scrapy/utils/ftp.py E302
|
||||
scrapy/utils/gz.py E305 E501 E302 W504
|
||||
scrapy/utils/http.py F403 F401 E226
|
||||
scrapy/utils/httpobj.py E302 E501
|
||||
scrapy/utils/iterators.py E501 E701
|
||||
scrapy/utils/job.py E302
|
||||
scrapy/utils/log.py E128 W503
|
||||
scrapy/utils/markup.py F403 F401 W292
|
||||
scrapy/utils/misc.py E501 E226
|
||||
scrapy/utils/multipart.py F403 F401 W292
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501 E302
|
||||
scrapy/utils/reactor.py E302 E226
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E302 E127 E501
|
||||
scrapy/utils/response.py E501 E302 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E271 E302 E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/template.py E302
|
||||
scrapy/utils/test.py E302 E501
|
||||
scrapy/utils/url.py E501 F403 F401 E128 F405
|
||||
# scrapy
|
||||
scrapy/__init__.py E402 E501
|
||||
scrapy/_monkeypatches.py W293
|
||||
scrapy/cmdline.py E502 E501
|
||||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E302 E501 E202
|
||||
scrapy/exceptions.py E302 E501
|
||||
scrapy/exporters.py E501 E261 E226
|
||||
scrapy/extension.py E302
|
||||
scrapy/interfaces.py E302 E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
scrapy/logformatter.py E501 W293
|
||||
scrapy/mail.py E402 E128 E501 E502
|
||||
scrapy/middleware.py E502 E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E302
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/robotstxt.py E302 E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py E225 F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py F401 E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123 F401
|
||||
tests/pipelines.py E302 F841 E226
|
||||
tests/spiders.py E302 E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501 E261
|
||||
tests/test_command_parse.py F401 E302 E501 E128 E303 E226
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py F401 E128 E501
|
||||
tests/test_contracts.py E501 E128 W293
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py E302 F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E711 E126 E226 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E713 E501 E302 E305 F401
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
|
||||
tests/test_engine.py E401 E501 E502 E128 E261
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_extension_telnet.py F401 F841
|
||||
tests/test_feedexport.py E501 F401 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E302 E501
|
||||
tests/test_http_request.py F401 E402 E501 E231 E261 E127 E128 W293 E502 E128 E502 E126 E123
|
||||
tests/test_http_response.py E501 E301 E502 E128 E265
|
||||
tests/test_item.py E701 E128 E231 F841 E306
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E231 E124
|
||||
tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E231 E122 E302
|
||||
tests/test_mail.py E302 E128 E501 E305
|
||||
tests/test_middleware.py E302 E501 E128
|
||||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
|
||||
tests/test_pipeline_images.py F401 F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E302 E305
|
||||
tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py F401 E501 E127
|
||||
tests/test_spider.py E501 F401
|
||||
tests/test_spidermiddleware.py E501 E226
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
|
||||
tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
|
||||
tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E302 E701 E741
|
||||
tests/test_utils_conf.py E501 E231 E303 E128
|
||||
tests/test_utils_console.py E302 E231
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_http.py E302 E501 E502 E128 W504
|
||||
tests/test_utils_httpobj.py E302
|
||||
tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
tests/test_utils_reqser.py F401 E501 E128
|
||||
tests/test_utils_request.py E302 E501 E128 E305
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E302 E731 E226
|
||||
tests/test_utils_sitemap.py E302 E128 E501 E124
|
||||
tests/test_utils_spider.py E261 E302 E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/mocks/dummydbm.py E302
|
||||
tests/test_cmdline/__init__.py E502 E501
|
||||
tests/test_cmdline/extensions.py E302
|
||||
tests/test_settings/__init__.py F401 E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501 E302
|
||||
tests/test_spiderloader/test_spiders/spider0.py E302
|
||||
tests/test_spiderloader/test_spiders/spider1.py E302
|
||||
tests/test_spiderloader/test_spiders/spider2.py E302
|
||||
tests/test_spiderloader/test_spiders/spider3.py E302
|
||||
tests/test_spiderloader/test_spiders/nested/spider4.py E302
|
||||
tests/test_utils_misc/__init__.py E501 E231
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
parsel>=1.5.0
|
||||
PyDispatcher>=2.0.5
|
||||
w3lib>=1.17.0
|
||||
protego>=0.1.15
|
||||
|
||||
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
|
||||
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
|
||||
cryptography>=2.0 # Earlier versions would fail to install
|
||||
|
||||
# Reference versions taken from
|
||||
# https://packages.ubuntu.com/xenial/python/
|
||||
# https://packages.ubuntu.com/xenial/zope/
|
||||
cssselect>=0.9.1
|
||||
lxml>=3.5.0
|
||||
service_identity>=16.0.0
|
||||
six>=1.10.0
|
||||
Twisted>=16.0.0
|
||||
zope.interface>=4.1.3
|
||||
|
|
@ -1 +1 @@
|
|||
1.7.0
|
||||
1.8.0
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ del pkgutil
|
|||
|
||||
# Check minimum required Python version
|
||||
import sys
|
||||
if sys.version_info < (2, 7):
|
||||
print("Scrapy %s requires Python 2.7" % __version__)
|
||||
if sys.version_info < (3, 5):
|
||||
print("Scrapy %s requires Python 3.5" % __version__)
|
||||
sys.exit(1)
|
||||
|
||||
# Ignore noisy twisted deprecation warnings
|
||||
|
|
|
|||
|
|
@ -96,4 +96,3 @@ class Command(ScrapyCommand):
|
|||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
|
||||
|
|
|
|||
|
|
@ -119,4 +119,3 @@ class Command(ScrapyCommand):
|
|||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'project')
|
||||
|
||||
|
|
|
|||
|
|
@ -30,4 +30,3 @@ class Command(ScrapyCommand):
|
|||
print(patt % (name, version))
|
||||
else:
|
||||
print("Scrapy %s" % scrapy.__version__)
|
||||
|
||||
|
|
|
|||
|
|
@ -112,4 +112,3 @@ class FTPDownloadHandler(object):
|
|||
httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
|
||||
return Response(url=request.url, status=httpcode, body=to_bytes(message))
|
||||
raise result.type(result.value)
|
||||
|
||||
|
|
|
|||
|
|
@ -157,4 +157,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
def gotHeaders(self, headers):
|
||||
self.headers_time = time()
|
||||
self.response_headers = headers
|
||||
|
||||
|
|
|
|||
|
|
@ -244,4 +244,3 @@ class Scraper(object):
|
|||
return self.signals.send_catch_log_deferred(
|
||||
signal=signals.item_scraped, item=output, response=response,
|
||||
spider=spider)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import logging
|
||||
from six.moves.urllib.parse import urljoin
|
||||
from six.moves.urllib.parse import urljoin, urlparse
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
|
|
@ -70,7 +70,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
|
|||
if 'Location' not in response.headers or response.status not in allowed_status:
|
||||
return response
|
||||
|
||||
location = safe_url_string(response.headers['location'])
|
||||
location = safe_url_string(response.headers['Location'])
|
||||
if response.headers['Location'].startswith(b'//'):
|
||||
request_scheme = urlparse(request.url).scheme
|
||||
location = request_scheme + '://' + location.lstrip('/')
|
||||
|
||||
redirected_url = urljoin(request.url, location)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,24 @@
|
|||
from __future__ import print_function
|
||||
import os
|
||||
|
||||
import gzip
|
||||
import logging
|
||||
from six.moves import cPickle as pickle
|
||||
import os
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
from importlib import import_module
|
||||
from time import time
|
||||
from warnings import warn
|
||||
from weakref import WeakKeyDictionary
|
||||
from email.utils import mktime_tz, parsedate_tz
|
||||
|
||||
from six.moves import cPickle as pickle
|
||||
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Headers, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.project import data_path
|
||||
from scrapy.utils.python import to_bytes, to_unicode, garbage_collect
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -342,75 +347,6 @@ class FilesystemCacheStorage(object):
|
|||
return pickle.load(f)
|
||||
|
||||
|
||||
class LeveldbCacheStorage(object):
|
||||
|
||||
def __init__(self, settings):
|
||||
import leveldb
|
||||
self._leveldb = leveldb
|
||||
self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
|
||||
self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
|
||||
self.db = None
|
||||
|
||||
def open_spider(self, spider):
|
||||
dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
|
||||
self.db = self._leveldb.LevelDB(dbpath)
|
||||
|
||||
logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider})
|
||||
|
||||
def close_spider(self, spider):
|
||||
# Do compactation each time to save space and also recreate files to
|
||||
# avoid them being removed in storages with timestamp-based autoremoval.
|
||||
self.db.CompactRange()
|
||||
del self.db
|
||||
garbage_collect()
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
url = data['url']
|
||||
status = data['status']
|
||||
headers = Headers(data['headers'])
|
||||
body = data['body']
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
response = respcls(url=url, headers=headers, status=status, body=body)
|
||||
return response
|
||||
|
||||
def store_response(self, spider, request, response):
|
||||
key = self._request_key(request)
|
||||
data = {
|
||||
'status': response.status,
|
||||
'url': response.url,
|
||||
'headers': dict(response.headers),
|
||||
'body': response.body,
|
||||
}
|
||||
batch = self._leveldb.WriteBatch()
|
||||
batch.Put(key + b'_data', pickle.dumps(data, protocol=2))
|
||||
batch.Put(key + b'_time', to_bytes(str(time())))
|
||||
self.db.Write(batch)
|
||||
|
||||
def _read_data(self, spider, request):
|
||||
key = self._request_key(request)
|
||||
try:
|
||||
ts = self.db.Get(key + b'_time')
|
||||
except KeyError:
|
||||
return # not found or invalid entry
|
||||
|
||||
if 0 < self.expiration_secs < time() - float(ts):
|
||||
return # expired
|
||||
|
||||
try:
|
||||
data = self.db.Get(key + b'_data')
|
||||
except KeyError:
|
||||
return # invalid entry
|
||||
else:
|
||||
return pickle.loads(data)
|
||||
|
||||
def _request_key(self, request):
|
||||
return to_bytes(request_fingerprint(request))
|
||||
|
||||
|
||||
|
||||
def parse_cachecontrol(header):
|
||||
"""Parse Cache-Control header
|
||||
|
||||
|
|
|
|||
|
|
@ -91,5 +91,3 @@ class Headers(CaselessDict):
|
|||
def __copy__(self):
|
||||
return self.__class__(self)
|
||||
copy = __copy__
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,4 +15,3 @@ class ISpiderLoader(Interface):
|
|||
|
||||
def find_by_request(request):
|
||||
"""Return the list of spiders names that can handle the given request"""
|
||||
|
||||
|
|
|
|||
|
|
@ -39,4 +39,3 @@ class Link(object):
|
|||
def __repr__(self):
|
||||
return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
|
||||
(self.url, self.text, self.fragment, self.nofollow)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,19 @@
|
|||
"""Item Loader
|
||||
"""
|
||||
Item Loader
|
||||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
|
||||
"""
|
||||
from collections import defaultdict
|
||||
|
||||
import six
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.loader.processors import Identity
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.utils.misc import arg_to_iter, extract_regex
|
||||
from scrapy.utils.python import flatten
|
||||
|
||||
from .common import wrap_loader_context
|
||||
from .processors import Identity
|
||||
|
||||
|
||||
class ItemLoader(object):
|
||||
|
||||
|
|
@ -33,10 +33,9 @@ class ItemLoader(object):
|
|||
self.parent = parent
|
||||
self._local_item = context['item'] = item
|
||||
self._local_values = defaultdict(list)
|
||||
# Preprocess values if item built from dict
|
||||
# Values need to be added to item._values if added them from dict (not with add_values)
|
||||
# values from initial item
|
||||
for field_name, value in item.items():
|
||||
self._values[field_name] = self._process_input_value(field_name, value)
|
||||
self._values[field_name] += arg_to_iter(value)
|
||||
|
||||
@property
|
||||
def _values(self):
|
||||
|
|
@ -132,8 +131,8 @@ class ItemLoader(object):
|
|||
try:
|
||||
return proc(self._values[field_name])
|
||||
except Exception as e:
|
||||
raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
|
||||
(field_name, self._values[field_name], type(e).__name__, str(e)))
|
||||
raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
|
||||
(field_name, self._values[field_name], type(e).__name__, str(e)))
|
||||
|
||||
def get_collected_values(self, field_name):
|
||||
return self._values[field_name]
|
||||
|
|
@ -141,15 +140,15 @@ class ItemLoader(object):
|
|||
def get_input_processor(self, field_name):
|
||||
proc = getattr(self, '%s_in' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor', \
|
||||
self.default_input_processor)
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor',
|
||||
self.default_input_processor)
|
||||
return proc
|
||||
|
||||
def get_output_processor(self, field_name):
|
||||
proc = getattr(self, '%s_out' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor', \
|
||||
self.default_output_processor)
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor',
|
||||
self.default_output_processor)
|
||||
return proc
|
||||
|
||||
def _process_input_value(self, field_name, value):
|
||||
|
|
@ -174,8 +173,8 @@ class ItemLoader(object):
|
|||
def _check_selector_method(self):
|
||||
if self.selector is None:
|
||||
raise RuntimeError("To use XPath or CSS selectors, "
|
||||
"%s must be instantiated with a selector "
|
||||
"or a response" % self.__class__.__name__)
|
||||
"%s must be instantiated with a selector "
|
||||
"or a response" % self.__class__.__name__)
|
||||
|
||||
def add_xpath(self, field_name, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class LogFormatter(object):
|
|||
* ``args`` should be a tuple or dict with the formatting placeholders for ``msg``.
|
||||
The final log message is computed as ``msg % args``.
|
||||
|
||||
Users can define their own ``LogFormatter`` class if they want to customise how
|
||||
Users can define their own ``LogFormatter`` class if they want to customize how
|
||||
each action is logged or if they want to omit it entirely. In order to omit
|
||||
logging an action the method must return ``None``.
|
||||
|
||||
|
|
|
|||
|
|
@ -133,4 +133,3 @@ class CSVFeedSpider(Spider):
|
|||
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
|
||||
response = self.adapt_response(response)
|
||||
return self.parse_rows(response)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,4 +29,3 @@ class InitSpider(Spider):
|
|||
spider
|
||||
"""
|
||||
return self.initialized()
|
||||
|
||||
|
|
|
|||
|
|
@ -80,5 +80,3 @@ class DummyStatsCollector(StatsCollector):
|
|||
|
||||
def min_value(self, key, value, spider=None):
|
||||
pass
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -34,4 +34,3 @@ def decode_chunked_transfer(chunked_body):
|
|||
body += t[:size]
|
||||
t = t[size+2:]
|
||||
return body
|
||||
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
"""
|
||||
if settings is None:
|
||||
if crawler is None:
|
||||
raise ValueError("Specifiy at least one of settings and crawler.")
|
||||
raise ValueError("Specify at least one of settings and crawler.")
|
||||
settings = crawler.settings
|
||||
if crawler and hasattr(objcls, 'from_crawler'):
|
||||
return objcls.from_crawler(crawler, *args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ from scrapy.utils.httpobj import urlparse_cached
|
|||
|
||||
|
||||
_fingerprint_cache = weakref.WeakKeyDictionary()
|
||||
def request_fingerprint(request, include_headers=None):
|
||||
def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
||||
"""
|
||||
Return the request fingerprint.
|
||||
|
||||
|
|
@ -42,15 +42,21 @@ def request_fingerprint(request, include_headers=None):
|
|||
the fingeprint. If you want to include specific headers use the
|
||||
include_headers argument, which is a list of Request headers to include.
|
||||
|
||||
Also, servers usually ignore fragments in urls when handling requests,
|
||||
so they are also ignored by default when calculating the fingerprint.
|
||||
If you want to include them, set the keep_fragments argument to True
|
||||
(for instance when handling requests with a headless browser).
|
||||
|
||||
"""
|
||||
if include_headers:
|
||||
include_headers = tuple(to_bytes(h.lower())
|
||||
for h in sorted(include_headers))
|
||||
cache = _fingerprint_cache.setdefault(request, {})
|
||||
if include_headers not in cache:
|
||||
cache_key = (include_headers, keep_fragments)
|
||||
if cache_key not in cache:
|
||||
fp = hashlib.sha1()
|
||||
fp.update(to_bytes(request.method))
|
||||
fp.update(to_bytes(canonicalize_url(request.url)))
|
||||
fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
|
||||
fp.update(request.body or b'')
|
||||
if include_headers:
|
||||
for hdr in include_headers:
|
||||
|
|
@ -58,8 +64,8 @@ def request_fingerprint(request, include_headers=None):
|
|||
fp.update(hdr)
|
||||
for v in request.headers.getlist(hdr):
|
||||
fp.update(v)
|
||||
cache[include_headers] = fp.hexdigest()
|
||||
return cache[include_headers]
|
||||
cache[cache_key] = fp.hexdigest()
|
||||
return cache[cache_key]
|
||||
|
||||
|
||||
def request_authenticate(request, username, password):
|
||||
|
|
|
|||
8
setup.py
8
setup.py
|
|
@ -50,22 +50,20 @@ setup(
|
|||
'License :: OSI Approved :: BSD License',
|
||||
'Operating System :: OS Independent',
|
||||
'Programming Language :: Python',
|
||||
'Programming Language :: Python :: 2',
|
||||
'Programming Language :: Python :: 2.7',
|
||||
'Programming Language :: Python :: 3',
|
||||
'Programming Language :: Python :: 3.5',
|
||||
'Programming Language :: Python :: 3.6',
|
||||
'Programming Language :: Python :: 3.7',
|
||||
'Programming Language :: Python :: 3.8',
|
||||
'Programming Language :: Python :: Implementation :: CPython',
|
||||
'Programming Language :: Python :: Implementation :: PyPy',
|
||||
'Topic :: Internet :: WWW/HTTP',
|
||||
'Topic :: Software Development :: Libraries :: Application Frameworks',
|
||||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
|
||||
python_requires='>=3.5',
|
||||
install_requires=[
|
||||
'Twisted>=16.0.0;python_version=="2.7"',
|
||||
'Twisted>=17.9.0;python_version>="3.5"',
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
'cssselect>=0.9.1',
|
||||
'lxml>=3.5.0',
|
||||
|
|
|
|||
|
|
@ -206,8 +206,7 @@ class MockServer():
|
|||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.proc.kill()
|
||||
self.proc.wait()
|
||||
time.sleep(0.2)
|
||||
self.proc.communicate()
|
||||
|
||||
def url(self, path, is_secure=False):
|
||||
host = self.http_address.replace('0.0.0.0', '127.0.0.1')
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
# Tests requirements
|
||||
brotlipy
|
||||
jmespath
|
||||
mitmproxy==0.10.1
|
||||
mock
|
||||
netlib==0.10.1
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-twisted
|
||||
pytest-xdist
|
||||
testfixtures
|
||||
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
ipython<6.0
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
# Tests requirements
|
||||
jmespath
|
||||
leveldb; sys_platform != "win32"
|
||||
pytest
|
||||
pytest-cov
|
||||
pytest-twisted
|
||||
|
|
|
|||
|
|
@ -12,4 +12,3 @@ class TestExtension(object):
|
|||
|
||||
class DummyExtension(object):
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import unittest
|
|||
import email.utils
|
||||
from contextlib import contextmanager
|
||||
import pytest
|
||||
import sys
|
||||
|
||||
from scrapy.http import Response, HtmlResponse, Request
|
||||
from scrapy.spiders import Spider
|
||||
|
|
@ -154,11 +155,6 @@ class FilesystemStorageGzipTest(FilesystemStorageTest):
|
|||
new_settings.setdefault('HTTPCACHE_GZIP', True)
|
||||
return super(FilesystemStorageTest, self)._get_settings(**new_settings)
|
||||
|
||||
class LeveldbStorageTest(DefaultStorageTest):
|
||||
|
||||
pytest.importorskip('leveldb')
|
||||
storage_class = 'scrapy.extensions.httpcache.LeveldbCacheStorage'
|
||||
|
||||
|
||||
class DummyPolicyTest(_BaseTest):
|
||||
|
||||
|
|
|
|||
|
|
@ -106,6 +106,22 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
del rsp.headers['Location']
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
def test_redirect_302_relative(self):
|
||||
url = 'http://www.example.com/302'
|
||||
url2 = '///i8n.example2.com/302'
|
||||
url3 = 'http://i8n.example2.com/302'
|
||||
req = Request(url, method='HEAD')
|
||||
rsp = Response(url, headers={'Location': url2}, status=302)
|
||||
|
||||
req2 = self.mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(req2, Request)
|
||||
self.assertEqual(req2.url, url3)
|
||||
self.assertEqual(req2.method, 'HEAD')
|
||||
|
||||
# response without Location header but with status code is 3XX should be ignored
|
||||
del rsp.headers['Location']
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
|
||||
def test_max_redirect_times(self):
|
||||
self.mw.max_redirect_times = 1
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
import unittest
|
||||
import six
|
||||
from functools import partial
|
||||
import unittest
|
||||
|
||||
import six
|
||||
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.processors import Join, Identity, TakeFirst, \
|
||||
Compose, MapCompose, SelectJmes
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.processors import (Compose, Identity, Join,
|
||||
MapCompose, SelectJmes, TakeFirst)
|
||||
from scrapy.selector import Selector
|
||||
|
||||
|
||||
# test items
|
||||
class NameItem(Item):
|
||||
|
|
@ -61,7 +63,7 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
il.add_value('name', u'marta')
|
||||
item = il.load_item()
|
||||
assert item is i
|
||||
self.assertEqual(item['summary'], u'lala')
|
||||
self.assertEqual(item['summary'], [u'lala'])
|
||||
self.assertEqual(item['name'], [u'marta'])
|
||||
|
||||
def test_load_item_using_custom_loader(self):
|
||||
|
|
@ -419,43 +421,6 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
self.assertEqual(item['url'], u'rabbit.hole')
|
||||
self.assertEqual(item['summary'], u'rabbithole')
|
||||
|
||||
def test_create_item_from_dict(self):
|
||||
class TestItem(Item):
|
||||
title = Field()
|
||||
|
||||
class TestItemLoader(ItemLoader):
|
||||
default_item_class = TestItem
|
||||
|
||||
input_item = {'title': 'Test item title 1'}
|
||||
il = TestItemLoader(item=input_item)
|
||||
# Getting output value mustn't remove value from item
|
||||
self.assertEqual(il.load_item(), {
|
||||
'title': 'Test item title 1',
|
||||
})
|
||||
self.assertEqual(il.get_output_value('title'), 'Test item title 1')
|
||||
self.assertEqual(il.load_item(), {
|
||||
'title': 'Test item title 1',
|
||||
})
|
||||
|
||||
input_item = {'title': 'Test item title 2'}
|
||||
il = TestItemLoader(item=input_item)
|
||||
# Values from dict must be added to item _values
|
||||
self.assertEqual(il._values.get('title'), 'Test item title 2')
|
||||
|
||||
input_item = {'title': [u'Test item title 3', u'Test item 4']}
|
||||
il = TestItemLoader(item=input_item)
|
||||
# Same rules must work for lists
|
||||
self.assertEqual(il._values.get('title'),
|
||||
[u'Test item title 3', u'Test item 4'])
|
||||
self.assertEqual(il.load_item(), {
|
||||
'title': [u'Test item title 3', u'Test item 4'],
|
||||
})
|
||||
self.assertEqual(il.get_output_value('title'),
|
||||
[u'Test item title 3', u'Test item 4'])
|
||||
self.assertEqual(il.load_item(), {
|
||||
'title': [u'Test item title 3', u'Test item 4'],
|
||||
})
|
||||
|
||||
def test_error_input_processor(self):
|
||||
class TestItem(Item):
|
||||
name = Field()
|
||||
|
|
@ -493,6 +458,220 @@ class BasicItemLoaderTest(unittest.TestCase):
|
|||
[u'marta', u'other'], Compose(float))
|
||||
|
||||
|
||||
class InitializationTestMixin(object):
|
||||
|
||||
item_class = None
|
||||
|
||||
def test_keep_single_value(self):
|
||||
"""Loaded item should contain values from the initial item"""
|
||||
input_item = self.item_class(name='foo')
|
||||
il = ItemLoader(item=input_item)
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {'name': ['foo']})
|
||||
|
||||
def test_keep_list(self):
|
||||
"""Loaded item should contain values from the initial item"""
|
||||
input_item = self.item_class(name=['foo', 'bar'])
|
||||
il = ItemLoader(item=input_item)
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
|
||||
|
||||
def test_add_value_singlevalue_singlevalue(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name='foo')
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value('name', 'bar')
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar']})
|
||||
|
||||
def test_add_value_singlevalue_list(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name='foo')
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value('name', ['item', 'loader'])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {'name': ['foo', 'item', 'loader']})
|
||||
|
||||
def test_add_value_list_singlevalue(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name=['foo', 'bar'])
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value('name', 'qwerty')
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
|
||||
|
||||
def test_add_value_list_list(self):
|
||||
"""Values added after initialization should be appended"""
|
||||
input_item = self.item_class(name=['foo', 'bar'])
|
||||
il = ItemLoader(item=input_item)
|
||||
il.add_value('name', ['item', 'loader'])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'item', 'loader']})
|
||||
|
||||
def test_get_output_value_singlevalue(self):
|
||||
"""Getting output value must not remove value from item"""
|
||||
input_item = self.item_class(name='foo')
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il.get_output_value('name'), ['foo'])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(loaded_item, dict({'name': ['foo']}))
|
||||
|
||||
def test_get_output_value_list(self):
|
||||
"""Getting output value must not remove value from item"""
|
||||
input_item = self.item_class(name=['foo', 'bar'])
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
|
||||
loaded_item = il.load_item()
|
||||
self.assertIsInstance(loaded_item, self.item_class)
|
||||
self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
|
||||
|
||||
def test_values_single(self):
|
||||
"""Values from initial item must be added to loader._values"""
|
||||
input_item = self.item_class(name='foo')
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il._values.get('name'), ['foo'])
|
||||
|
||||
def test_values_list(self):
|
||||
"""Values from initial item must be added to loader._values"""
|
||||
input_item = self.item_class(name=['foo', 'bar'])
|
||||
il = ItemLoader(item=input_item)
|
||||
self.assertEqual(il._values.get('name'), ['foo', 'bar'])
|
||||
|
||||
|
||||
class InitializationFromDictTest(InitializationTestMixin, unittest.TestCase):
|
||||
item_class = dict
|
||||
|
||||
|
||||
class InitializationFromItemTest(InitializationTestMixin, unittest.TestCase):
|
||||
item_class = NameItem
|
||||
|
||||
|
||||
class BaseNoInputReprocessingLoader(ItemLoader):
|
||||
title_in = MapCompose(str.upper)
|
||||
title_out = TakeFirst()
|
||||
|
||||
|
||||
class NoInputReprocessingDictLoader(BaseNoInputReprocessingLoader):
|
||||
default_item_class = dict
|
||||
|
||||
|
||||
class NoInputReprocessingFromDictTest(unittest.TestCase):
|
||||
"""
|
||||
Loaders initialized from loaded items must not reprocess fields (dict instances)
|
||||
"""
|
||||
def test_avoid_reprocessing_with_initial_values_single(self):
|
||||
il = NoInputReprocessingDictLoader(item=dict(title='foo'))
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, dict(title='foo'))
|
||||
self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
|
||||
|
||||
def test_avoid_reprocessing_with_initial_values_list(self):
|
||||
il = NoInputReprocessingDictLoader(item=dict(title=['foo', 'bar']))
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, dict(title='foo'))
|
||||
self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='foo'))
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_single(self):
|
||||
il = NoInputReprocessingDictLoader()
|
||||
il.add_value('title', 'foo')
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, dict(title='FOO'))
|
||||
self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_list(self):
|
||||
il = NoInputReprocessingDictLoader()
|
||||
il.add_value('title', ['foo', 'bar'])
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, dict(title='FOO'))
|
||||
self.assertEqual(NoInputReprocessingDictLoader(item=il_loaded).load_item(), dict(title='FOO'))
|
||||
|
||||
|
||||
class NoInputReprocessingItem(Item):
|
||||
title = Field()
|
||||
|
||||
|
||||
class NoInputReprocessingItemLoader(BaseNoInputReprocessingLoader):
|
||||
default_item_class = NoInputReprocessingItem
|
||||
|
||||
|
||||
class NoInputReprocessingFromItemTest(unittest.TestCase):
|
||||
"""
|
||||
Loaders initialized from loaded items must not reprocess fields (BaseItem instances)
|
||||
"""
|
||||
def test_avoid_reprocessing_with_initial_values_single(self):
|
||||
il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title='foo'))
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {'title': 'foo'})
|
||||
self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
|
||||
|
||||
def test_avoid_reprocessing_with_initial_values_list(self):
|
||||
il = NoInputReprocessingItemLoader(item=NoInputReprocessingItem(title=['foo', 'bar']))
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {'title': 'foo'})
|
||||
self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'foo'})
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_single(self):
|
||||
il = NoInputReprocessingItemLoader()
|
||||
il.add_value('title', 'FOO')
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {'title': 'FOO'})
|
||||
self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
|
||||
|
||||
def test_avoid_reprocessing_without_initial_values_list(self):
|
||||
il = NoInputReprocessingItemLoader()
|
||||
il.add_value('title', ['foo', 'bar'])
|
||||
il_loaded = il.load_item()
|
||||
self.assertEqual(il_loaded, {'title': 'FOO'})
|
||||
self.assertEqual(NoInputReprocessingItemLoader(item=il_loaded).load_item(), {'title': 'FOO'})
|
||||
|
||||
|
||||
class TestOutputProcessorDict(unittest.TestCase):
|
||||
def test_output_processor(self):
|
||||
|
||||
class TempDict(dict):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(TempDict, self).__init__(self, *args, **kwargs)
|
||||
self.setdefault('temp', 0.3)
|
||||
|
||||
class TempLoader(ItemLoader):
|
||||
default_item_class = TempDict
|
||||
default_input_processor = Identity()
|
||||
default_output_processor = Compose(TakeFirst())
|
||||
|
||||
loader = TempLoader()
|
||||
item = loader.load_item()
|
||||
self.assertIsInstance(item, TempDict)
|
||||
self.assertEqual(dict(item), {'temp': 0.3})
|
||||
|
||||
|
||||
class TestOutputProcessorItem(unittest.TestCase):
|
||||
def test_output_processor(self):
|
||||
|
||||
class TempItem(Item):
|
||||
temp = Field()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(TempItem, self).__init__(self, *args, **kwargs)
|
||||
self.setdefault('temp', 0.3)
|
||||
|
||||
class TempLoader(ItemLoader):
|
||||
default_item_class = TempItem
|
||||
default_input_processor = Identity()
|
||||
default_output_processor = Compose(TakeFirst())
|
||||
|
||||
loader = TempLoader()
|
||||
item = loader.load_item()
|
||||
self.assertIsInstance(item, TempItem)
|
||||
self.assertEqual(dict(item), {'temp': 0.3})
|
||||
|
||||
|
||||
class ProcessorsTest(unittest.TestCase):
|
||||
|
||||
def test_take_first(self):
|
||||
|
|
@ -523,7 +702,8 @@ class ProcessorsTest(unittest.TestCase):
|
|||
self.assertRaises(ValueError, proc, 'hello')
|
||||
|
||||
def test_mapcompose(self):
|
||||
filter_world = lambda x: None if x == 'world' else x
|
||||
def filter_world(x):
|
||||
return None if x == 'world' else x
|
||||
proc = MapCompose(filter_world, six.text_type.upper)
|
||||
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
|
||||
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
|
||||
|
|
@ -535,7 +715,6 @@ class ProcessorsTest(unittest.TestCase):
|
|||
self.assertRaises(ValueError, proc, 'hello')
|
||||
|
||||
|
||||
|
||||
class SelectortemLoaderTest(unittest.TestCase):
|
||||
response = HtmlResponse(url="", encoding='utf-8', body=b"""
|
||||
<html>
|
||||
|
|
@ -672,7 +851,7 @@ class SelectortemLoaderTest(unittest.TestCase):
|
|||
|
||||
self.assertEqual(l.get_css(['p::text', 'div::text']), [u'paragraph', 'marta'])
|
||||
self.assertEqual(l.get_css(['a::attr(href)', 'img::attr(src)']),
|
||||
[u'http://www.scrapy.org', u'/images/logo.png'])
|
||||
[u'http://www.scrapy.org', u'/images/logo.png'])
|
||||
|
||||
def test_replace_css_multi_fields(self):
|
||||
l = TestItemLoader(response=self.response)
|
||||
|
|
@ -720,7 +899,7 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
|
||||
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
|
||||
|
|
@ -735,7 +914,7 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
|
||||
self.assertEqual(l.get_output_value('name'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_div'), [u'<div id="id">marta</div>'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
self.assertEqual(l.get_output_value('name_value'), [u'marta'])
|
||||
|
||||
self.assertEqual(l.get_output_value('name'), nl.get_output_value('name'))
|
||||
self.assertEqual(l.get_output_value('name_div'), nl.get_output_value('name_div'))
|
||||
|
|
@ -791,28 +970,28 @@ class SubselectorLoaderTest(unittest.TestCase):
|
|||
|
||||
|
||||
class SelectJmesTestCase(unittest.TestCase):
|
||||
test_list_equals = {
|
||||
'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
|
||||
'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
|
||||
'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
|
||||
'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
|
||||
'dict': (
|
||||
'foo.bar[*].name',
|
||||
{"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
|
||||
['one', 'two']
|
||||
),
|
||||
'list': ('[1]', [1, 2], 2)
|
||||
}
|
||||
test_list_equals = {
|
||||
'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
|
||||
'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None),
|
||||
'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}),
|
||||
'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"),
|
||||
'dict': (
|
||||
'foo.bar[*].name',
|
||||
{"foo": {"bar": [{"name": "one"}, {"name": "two"}]}},
|
||||
['one', 'two']
|
||||
),
|
||||
'list': ('[1]', [1, 2], 2)
|
||||
}
|
||||
|
||||
def test_output(self):
|
||||
for l in self.test_list_equals:
|
||||
expr, test_list, expected = self.test_list_equals[l]
|
||||
test = SelectJmes(expr)(test_list)
|
||||
self.assertEqual(
|
||||
test,
|
||||
expected,
|
||||
msg='test "{}" got {} expected {}'.format(l, test, expected)
|
||||
)
|
||||
def test_output(self):
|
||||
for l in self.test_list_equals:
|
||||
expr, test_list, expected = self.test_list_equals[l]
|
||||
test = SelectJmes(expr)(test_list)
|
||||
self.assertEqual(
|
||||
test,
|
||||
expected,
|
||||
msg='test "{}" got {} expected {}'.format(l, test, expected)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -2,4 +2,3 @@
|
|||
TEST_DEFAULT = 'defvalue'
|
||||
|
||||
TEST_DICT = {'key': 'val'}
|
||||
|
||||
|
|
|
|||
|
|
@ -40,4 +40,3 @@ class TestDepthMiddleware(TestCase):
|
|||
|
||||
def tearDown(self):
|
||||
self.stats.close_spider(self.spider, '')
|
||||
|
||||
|
|
|
|||
|
|
@ -18,4 +18,3 @@ class TestUrlLengthMiddleware(TestCase):
|
|||
spider = Spider('foo')
|
||||
out = list(mw.process_spider_output(res, reqs, spider))
|
||||
self.assertEqual(out, [short_url_req])
|
||||
|
||||
|
|
|
|||
|
|
@ -244,4 +244,3 @@ class SequenceExcludeTest(unittest.TestCase):
|
|||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
|
|
|||
|
|
@ -16,5 +16,3 @@ class ChunkedTest(unittest.TestCase):
|
|||
"This is the data in the first chunk\r\n" +
|
||||
"and this is the second one\r\n" +
|
||||
"consequence")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class UtilsRequestTest(unittest.TestCase):
|
|||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
|
||||
# make sure caching is working
|
||||
self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][None])
|
||||
self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
|
||||
|
||||
r1 = Request("http://www.example.com/members/offers.html")
|
||||
r2 = Request("http://www.example.com/members/offers.html")
|
||||
|
|
@ -42,6 +42,13 @@ class UtilsRequestTest(unittest.TestCase):
|
|||
self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
|
||||
request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
|
||||
|
||||
r1 = Request("http://www.example.com/test.html")
|
||||
r2 = Request("http://www.example.com/test.html#fragment")
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
|
||||
self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
|
||||
self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
|
||||
self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
|
||||
|
||||
r1 = Request("http://www.example.com")
|
||||
r2 = Request("http://www.example.com", method='POST')
|
||||
r3 = Request("http://www.example.com", method='POST', body=b'request body')
|
||||
|
|
|
|||
|
|
@ -34,4 +34,3 @@ class UtilsSpidersTestCase(unittest.TestCase):
|
|||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
|
|
|||
73
tox.ini
73
tox.ini
|
|
@ -4,18 +4,16 @@
|
|||
# and then run "tox" from this directory.
|
||||
|
||||
[tox]
|
||||
envlist = py27
|
||||
envlist = py35
|
||||
|
||||
[testenv]
|
||||
deps =
|
||||
-ctests/constraints.txt
|
||||
-rrequirements-py2.txt
|
||||
-rrequirements-py3.txt
|
||||
-rtests/requirements-py3.txt
|
||||
# Extras
|
||||
botocore>=1.3.23
|
||||
google-cloud-storage
|
||||
leveldb
|
||||
Pillow>=3.4.2
|
||||
-rtests/requirements-py2.txt
|
||||
passenv =
|
||||
S3_TEST_FILE_URI
|
||||
AWS_ACCESS_KEY_ID
|
||||
|
|
@ -25,42 +23,8 @@ passenv =
|
|||
commands =
|
||||
py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
|
||||
|
||||
[testenv:py27-pinned]
|
||||
basepython = python2.7
|
||||
deps =
|
||||
-ctests/constraints.txt
|
||||
cryptography==2.0
|
||||
cssselect==0.9.1
|
||||
lxml==3.5.0
|
||||
parsel==1.5.0
|
||||
Protego==0.1.15
|
||||
PyDispatcher==2.0.5
|
||||
pyOpenSSL==16.2.0
|
||||
queuelib==1.4.2
|
||||
service_identity==16.0.0
|
||||
six==1.10.0
|
||||
Twisted==16.0.0
|
||||
w3lib==1.17.0
|
||||
zope.interface==4.1.3
|
||||
-rtests/requirements-py2.txt
|
||||
# Extras
|
||||
botocore==1.3.23
|
||||
Pillow==3.4.2
|
||||
|
||||
[testenv:pypy]
|
||||
basepython = pypy
|
||||
commands =
|
||||
py.test {posargs:scrapy tests}
|
||||
|
||||
[testenv:py35]
|
||||
basepython = python3.5
|
||||
deps =
|
||||
-ctests/constraints.txt
|
||||
-rrequirements-py3.txt
|
||||
-rtests/requirements-py3.txt
|
||||
# Extras
|
||||
botocore>=1.3.23
|
||||
Pillow>=3.4.2
|
||||
|
||||
[testenv:py35-pinned]
|
||||
basepython = python3.5
|
||||
|
|
@ -86,22 +50,30 @@ deps =
|
|||
|
||||
[testenv:py36]
|
||||
basepython = python3.6
|
||||
deps = {[testenv:py35]deps}
|
||||
|
||||
[testenv:py37]
|
||||
basepython = python3.7
|
||||
deps = {[testenv:py35]deps}
|
||||
|
||||
[testenv:py38]
|
||||
basepython = python3.8
|
||||
|
||||
[testenv:pypy3]
|
||||
basepython = pypy3
|
||||
deps = {[testenv:py35]deps}
|
||||
commands =
|
||||
py.test {posargs:scrapy tests}
|
||||
|
||||
[testenv:pylint]
|
||||
basepython = python3.7
|
||||
[testenv:flake8]
|
||||
basepython = python3.8
|
||||
deps =
|
||||
{[testenv:py35]deps}
|
||||
{[testenv]deps}
|
||||
pytest-flake8
|
||||
commands =
|
||||
py.test --flake8 {posargs:scrapy tests}
|
||||
|
||||
[testenv:pylint]
|
||||
basepython = python3.8
|
||||
deps =
|
||||
{[testenv]deps}
|
||||
# Optional dependencies
|
||||
boto
|
||||
reppy
|
||||
|
|
@ -135,15 +107,8 @@ deps = {[docs]deps}
|
|||
commands =
|
||||
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
|
||||
|
||||
[testenv:py37-extra-deps]
|
||||
basepython = python3.7
|
||||
deps =
|
||||
{[testenv:py35]deps}
|
||||
reppy
|
||||
robotexclusionrulesparser
|
||||
|
||||
[testenv:py27-extra-deps]
|
||||
basepython = python2.7
|
||||
[testenv:py38-extra-deps]
|
||||
basepython = python3.8
|
||||
deps =
|
||||
{[testenv]deps}
|
||||
reppy
|
||||
|
|
|
|||
Loading…
Reference in New Issue