mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names
This commit is contained in:
Adrián Chaves
commit
5138f9a965
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.1.0
|
||||
current_version = 2.3.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ htmlcov/
|
|||
.pytest_cache/
|
||||
.coverage.*
|
||||
.cache/
|
||||
.mypy_cache/
|
||||
|
||||
# Windows
|
||||
Thumbs.db
|
||||
|
|
|
|||
18
.travis.yml
18
.travis.yml
|
|
@ -15,20 +15,28 @@ matrix:
|
|||
python: 3.8
|
||||
- env: TOXENV=docs
|
||||
python: 3.7 # Keep in sync with .readthedocs.yml
|
||||
- env: TOXENV=typing
|
||||
python: 3.8
|
||||
|
||||
- env: TOXENV=pypy3
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5.2
|
||||
- env: TOXENV=asyncio
|
||||
- env: TOXENV=asyncio-pinned
|
||||
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
|
||||
- env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
|
|
@ -40,9 +48,9 @@ matrix:
|
|||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
variables:
|
||||
TOXENV: py
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
strategy:
|
||||
matrix:
|
||||
Python35:
|
||||
python.version: '3.5'
|
||||
TOXENV: windows-pinned
|
||||
Python36:
|
||||
python.version: '3.6'
|
||||
Python37:
|
||||
python.version: '3.7'
|
||||
Python38:
|
||||
python.version: '3.8'
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
displayName: 'Use Python $(python.version)'
|
||||
- script: |
|
||||
pip install -U tox twine wheel codecov
|
||||
tox
|
||||
displayName: 'Run test suite'
|
||||
|
|
@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
|
|||
def is_setting_index(node):
|
||||
if node.tagname == 'index':
|
||||
# index entries for setting directives look like:
|
||||
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
|
||||
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
|
||||
entry_type, info, refid = node['entries'][0][:3]
|
||||
return entry_type == 'pair' and info.endswith('; setting')
|
||||
return False
|
||||
|
|
|
|||
13
docs/conf.py
13
docs/conf.py
|
|
@ -284,6 +284,7 @@ intersphinx_mapping = {
|
|||
'attrs': ('https://www.attrs.org/en/stable/', None),
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
|
||||
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
|
||||
|
|
@ -305,3 +306,15 @@ hoverxref_role_types = {
|
|||
"ref": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.connect('autodoc-skip-member', maybe_skip_member)
|
||||
|
||||
|
||||
def maybe_skip_member(app, what, name, obj, skip, options):
|
||||
if not skip:
|
||||
# autodocs was generating a text "alias of" for the following members
|
||||
# https://github.com/sphinx-doc/sphinx/issues/4422
|
||||
return name in {'default_item_class', 'default_selector_class'}
|
||||
return skip
|
||||
|
|
|
|||
|
|
@ -155,6 +155,9 @@ Finally, try to keep aesthetic changes (:pep:`8` compliance, unused imports
|
|||
removal, etc) in separate commits from functional changes. This will make pull
|
||||
requests easier to review and more likely to get merged.
|
||||
|
||||
|
||||
.. _coding-style:
|
||||
|
||||
Coding style
|
||||
============
|
||||
|
||||
|
|
@ -163,7 +166,7 @@ Scrapy:
|
|||
|
||||
* Unless otherwise specified, follow :pep:`8`.
|
||||
|
||||
* It's OK to use lines longer than 80 chars if it improves the code
|
||||
* It's OK to use lines longer than 79 chars if it improves the code
|
||||
readability.
|
||||
|
||||
* Don't put your name in the code you contribute; git provides enough
|
||||
|
|
|
|||
14
docs/faq.rst
14
docs/faq.rst
|
|
@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
|
||||
.. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
|
||||
|
||||
.. _faq-python-versions:
|
||||
|
||||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 3.5.2+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
Python 2 support was dropped in Scrapy 2.0.
|
||||
|
||||
.. note::
|
||||
For Python 3 support on Windows, it is recommended to use
|
||||
Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
|
||||
|
||||
Did Scrapy "steal" X from Django?
|
||||
---------------------------------
|
||||
|
|
|
|||
|
|
@ -4,12 +4,18 @@
|
|||
Installation guide
|
||||
==================
|
||||
|
||||
.. _faq-python-versions:
|
||||
|
||||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
|
||||
the PyPy 5.9+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
|
||||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 3.5.2 or above under CPython (default Python
|
||||
implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
|
||||
and macOS.
|
||||
|
|
|
|||
328
docs/news.rst
328
docs/news.rst
|
|
@ -3,6 +3,334 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.3.0:
|
||||
|
||||
Scrapy 2.3.0 (2020-08-04)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
|
||||
Storage <topics-feed-storage-gcs>` as a storage backend
|
||||
|
||||
* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
|
||||
output items in batches of up to the specified number of items.
|
||||
|
||||
It also serves as a workaround for :ref:`delayed file delivery
|
||||
<delayed-file-delivery>`, which causes Scrapy to only start item delivery
|
||||
after the crawl has finished when using certain storage backends
|
||||
(:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
|
||||
and now :ref:`GCS <topics-feed-storage-gcs>`).
|
||||
|
||||
* The base implementation of :ref:`item loaders <topics-loaders>` has been
|
||||
moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
|
||||
allowing usage from outside Scrapy and a separate release schedule
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Removed the following classes and their parent modules from
|
||||
``scrapy.linkextractors``:
|
||||
|
||||
* ``htmlparser.HtmlParserLinkExtractor``
|
||||
* ``regex.RegexLinkExtractor``
|
||||
* ``sgml.BaseSgmlLinkExtractor``
|
||||
* ``sgml.SgmlLinkExtractor``
|
||||
|
||||
Use
|
||||
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
instead (:issue:`4356`, :issue:`4679`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
|
||||
(:issue:`4683`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
|
||||
Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
|
||||
|
||||
* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
|
||||
(:issue:`4250`, :issue:`4434`)
|
||||
|
||||
* The :command:`parse` command now allows specifying an output file
|
||||
(:issue:`4317`, :issue:`4377`)
|
||||
|
||||
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
|
||||
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
|
||||
``--data-raw`` (:issue:`4612`)
|
||||
|
||||
* A ``parse`` callback may now be used in built-in spider subclasses, such
|
||||
as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
|
||||
:issue:`781`, :issue:`4254` )
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
|
||||
:ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
|
||||
<attrs-items>` (:issue:`4667`, :issue:`4668`)
|
||||
|
||||
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
|
||||
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
|
||||
method to ``POST`` when a request body is specified and no request method
|
||||
is specified (:issue:`4612`)
|
||||
|
||||
* The processing of ANSI escape sequences in enabled in Windows 10.0.14393
|
||||
and later, where it is required for colored output (:issue:`4393`,
|
||||
:issue:`4403`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Updated the `OpenSSL cipher list format`_ link in the documentation about
|
||||
the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
|
||||
|
||||
* Simplified the code example in :ref:`topics-loaders-dataclass`
|
||||
(:issue:`4652`)
|
||||
|
||||
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
* The base implementation of :ref:`item loaders <topics-loaders>` has been
|
||||
moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
|
||||
:issue:`4516`)
|
||||
|
||||
* Fixed a silenced error in some scheduler tests (:issue:`4644`,
|
||||
:issue:`4645`)
|
||||
|
||||
* Renewed the localhost certificate used for SSL tests (:issue:`4650`)
|
||||
|
||||
* Removed cookie-handling code specific to Python 2 (:issue:`4682`)
|
||||
|
||||
* Stopped using Python 2 unicode literal syntax (:issue:`4704`)
|
||||
|
||||
* Stopped using a backlash for line continuation (:issue:`4673`)
|
||||
|
||||
* Removed unneeded entries from the MyPy exception list (:issue:`4690`)
|
||||
|
||||
* Automated tests now pass on Windows as part of our continuous integration
|
||||
system (:issue:`4458`)
|
||||
|
||||
* Automated tests now pass on the latest PyPy version for supported Python
|
||||
versions in our continuous integration system (:issue:`4504`)
|
||||
|
||||
|
||||
.. _release-2.2.1:
|
||||
|
||||
Scrapy 2.2.1 (2020-07-17)
|
||||
-------------------------
|
||||
|
||||
* The :command:`startproject` command no longer makes unintended changes to
|
||||
the permissions of files in the destination folder, such as removing
|
||||
execution permissions (:issue:`4662`, :issue:`4666`)
|
||||
|
||||
|
||||
.. _release-2.2.0:
|
||||
|
||||
Scrapy 2.2.0 (2020-06-24)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* Python 3.5.2+ is required now
|
||||
* :ref:`dataclass objects <dataclass-items>` and
|
||||
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
|
||||
<item-types>`
|
||||
* New :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
|
||||
* New :signal:`bytes_received` signal that allows canceling response download
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` fixes
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Support for Python 3.5.0 and 3.5.1 has been dropped; Scrapy now refuses to
|
||||
run with a Python version lower than 3.5.2, which introduced
|
||||
:class:`typing.Type` (:issue:`4615`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* :meth:`TextResponse.body_as_unicode
|
||||
<scrapy.http.TextResponse.body_as_unicode>` is now deprecated, use
|
||||
:attr:`TextResponse.text <scrapy.http.TextResponse.text>` instead
|
||||
(:issue:`4546`, :issue:`4555`, :issue:`4579`)
|
||||
|
||||
* :class:`scrapy.item.BaseItem` is now deprecated, use
|
||||
:class:`scrapy.item.Item` instead (:issue:`4534`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* :ref:`dataclass objects <dataclass-items>` and
|
||||
:ref:`attrs objects <attrs-items>` are now valid :ref:`item types
|
||||
<item-types>`, and a new itemadapter_ library makes it easy to
|
||||
write code that :ref:`supports any item type <supporting-item-types>`
|
||||
(:issue:`2749`, :issue:`2807`, :issue:`3761`, :issue:`3881`, :issue:`4642`)
|
||||
|
||||
* A new :meth:`TextResponse.json <scrapy.http.TextResponse.json>` method
|
||||
allows to deserialize JSON responses (:issue:`2444`, :issue:`4460`,
|
||||
:issue:`4574`)
|
||||
|
||||
* A new :signal:`bytes_received` signal allows monitoring response download
|
||||
progress and :ref:`stopping downloads <topics-stop-response-download>`
|
||||
(:issue:`4205`, :issue:`4559`)
|
||||
|
||||
* The dictionaries in the result list of a :ref:`media pipeline
|
||||
<topics-media-pipeline>` now include a new key, ``status``, which indicates
|
||||
if the file was downloaded or, if the file was not downloaded, why it was
|
||||
not downloaded; see :meth:`FilesPipeline.get_media_requests
|
||||
<scrapy.pipelines.files.FilesPipeline.get_media_requests>` for more
|
||||
information (:issue:`2893`, :issue:`4486`)
|
||||
|
||||
* When using :ref:`Google Cloud Storage <media-pipeline-gcs>` for
|
||||
a :ref:`media pipeline <topics-media-pipeline>`, a warning is now logged if
|
||||
the configured credentials do not grant the required permissions
|
||||
(:issue:`4346`, :issue:`4508`)
|
||||
|
||||
* :ref:`Link extractors <topics-link-extractors>` are now serializable,
|
||||
as long as you do not use :ref:`lambdas <lambda>` for parameters; for
|
||||
example, you can now pass link extractors in :attr:`Request.cb_kwargs
|
||||
<scrapy.http.Request.cb_kwargs>` or
|
||||
:attr:`Request.meta <scrapy.http.Request.meta>` when :ref:`persisting
|
||||
scheduled requests <topics-jobs>` (:issue:`4554`)
|
||||
|
||||
* Upgraded the :ref:`pickle protocol <pickle-protocols>` that Scrapy uses
|
||||
from protocol 2 to protocol 4, improving serialization capabilities and
|
||||
performance (:issue:`4135`, :issue:`4541`)
|
||||
|
||||
* :func:`scrapy.utils.misc.create_instance` now raises a :exc:`TypeError`
|
||||
exception if the resulting instance is ``None`` (:issue:`4528`,
|
||||
:issue:`4532`)
|
||||
|
||||
.. _itemadapter: https://github.com/scrapy/itemadapter
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
|
||||
discards cookies defined in :attr:`Request.headers
|
||||
<scrapy.http.Request.headers>` (:issue:`1992`, :issue:`2400`)
|
||||
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` no longer
|
||||
re-encodes cookies defined as :class:`bytes` in the ``cookies`` parameter
|
||||
of the ``__init__`` method of :class:`~scrapy.http.Request`
|
||||
(:issue:`2400`, :issue:`3575`)
|
||||
|
||||
* When :setting:`FEEDS` defines multiple URIs, :setting:`FEED_STORE_EMPTY` is
|
||||
``False`` and the crawl yields no items, Scrapy no longer stops feed
|
||||
exports after the first URI (:issue:`4621`, :issue:`4626`)
|
||||
|
||||
* :class:`~scrapy.spiders.Spider` callbacks defined using :doc:`coroutine
|
||||
syntax <topics/coroutines>` no longer need to return an iterable, and may
|
||||
instead return a :class:`~scrapy.http.Request` object, an
|
||||
:ref:`item <topics-items>`, or ``None`` (:issue:`4609`)
|
||||
|
||||
* The :command:`startproject` command now ensures that the generated project
|
||||
folders and files have the right permissions (:issue:`4604`)
|
||||
|
||||
* Fix a :exc:`KeyError` exception being sometimes raised from
|
||||
:class:`scrapy.utils.datatypes.LocalWeakReferencedCache` (:issue:`4597`,
|
||||
:issue:`4599`)
|
||||
|
||||
* When :setting:`FEEDS` defines multiple URIs, log messages about items being
|
||||
stored now contain information from the corresponding feed, instead of
|
||||
always containing information about only one of the feeds (:issue:`4619`,
|
||||
:issue:`4629`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Added a new section about :ref:`accessing cb_kwargs from errbacks
|
||||
<errback-cb_kwargs>` (:issue:`4598`, :issue:`4634`)
|
||||
|
||||
* Covered chompjs_ in :ref:`topics-parsing-javascript` (:issue:`4556`,
|
||||
:issue:`4562`)
|
||||
|
||||
* Removed from :doc:`topics/coroutines` the warning about the API being
|
||||
experimental (:issue:`4511`, :issue:`4513`)
|
||||
|
||||
* Removed references to unsupported versions of :doc:`Twisted
|
||||
<twisted:index>` (:issue:`4533`)
|
||||
|
||||
* Updated the description of the :ref:`screenshot pipeline example
|
||||
<ScreenshotPipeline>`, which now uses :doc:`coroutine syntax
|
||||
<topics/coroutines>` instead of returning a
|
||||
:class:`~twisted.internet.defer.Deferred` (:issue:`4514`, :issue:`4593`)
|
||||
|
||||
* Removed a misleading import line from the
|
||||
:func:`scrapy.utils.log.configure_logging` code example (:issue:`4510`,
|
||||
:issue:`4587`)
|
||||
|
||||
* The display-on-hover behavior of internal documentation references now also
|
||||
covers links to :ref:`commands <topics-commands>`, :attr:`Request.meta
|
||||
<scrapy.http.Request.meta>` keys, :ref:`settings <topics-settings>` and
|
||||
:ref:`signals <topics-signals>` (:issue:`4495`, :issue:`4563`)
|
||||
|
||||
* It is again possible to download the documentation for offline reading
|
||||
(:issue:`4578`, :issue:`4585`)
|
||||
|
||||
* Removed backslashes preceding ``*args`` and ``**kwargs`` in some function
|
||||
and method signatures (:issue:`4592`, :issue:`4596`)
|
||||
|
||||
.. _chompjs: https://github.com/Nykakin/chompjs
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Adjusted the code base further to our :ref:`style guidelines
|
||||
<coding-style>` (:issue:`4237`, :issue:`4525`, :issue:`4538`,
|
||||
:issue:`4539`, :issue:`4540`, :issue:`4542`, :issue:`4543`, :issue:`4544`,
|
||||
:issue:`4545`, :issue:`4557`, :issue:`4558`, :issue:`4566`, :issue:`4568`,
|
||||
:issue:`4572`)
|
||||
|
||||
* Removed remnants of Python 2 support (:issue:`4550`, :issue:`4553`,
|
||||
:issue:`4568`)
|
||||
|
||||
* Improved code sharing between the :command:`crawl` and :command:`runspider`
|
||||
commands (:issue:`4548`, :issue:`4552`)
|
||||
|
||||
* Replaced ``chain(*iterable)`` with ``chain.from_iterable(iterable)``
|
||||
(:issue:`4635`)
|
||||
|
||||
* You may now run the :mod:`asyncio` tests with Tox on any Python version
|
||||
(:issue:`4521`)
|
||||
|
||||
* Updated test requirements to reflect an incompatibility with pytest 5.4 and
|
||||
5.4.1 (:issue:`4588`)
|
||||
|
||||
* Improved :class:`~scrapy.spiderloader.SpiderLoader` test coverage for
|
||||
scenarios involving duplicate spider names (:issue:`4549`, :issue:`4560`)
|
||||
|
||||
* Configured Travis CI to also run the tests with Python 3.5.2
|
||||
(:issue:`4518`, :issue:`4615`)
|
||||
|
||||
* Added a `Pylint <https://www.pylint.org/>`_ job to Travis CI
|
||||
(:issue:`3727`)
|
||||
|
||||
* Added a `Mypy <http://mypy-lang.org/>`_ job to Travis CI (:issue:`4637`)
|
||||
|
||||
* Made use of set literals in tests (:issue:`4573`)
|
||||
|
||||
* Cleaned up the Travis CI configuration (:issue:`4517`, :issue:`4519`,
|
||||
:issue:`4522`, :issue:`4537`)
|
||||
|
||||
|
||||
.. _release-2.1.0:
|
||||
|
||||
Scrapy 2.1.0 (2020-04-24)
|
||||
|
|
|
|||
|
|
@ -468,7 +468,7 @@ Supported options:
|
|||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||
response
|
||||
|
||||
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
|
||||
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
|
||||
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
|
||||
|
||||
* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
|
||||
|
|
@ -491,6 +491,10 @@ Supported options:
|
|||
|
||||
* ``--verbose`` or ``-v``: display information for each depth level
|
||||
|
||||
* ``--output`` or ``-o``: dump scraped items to a file
|
||||
|
||||
.. versionadded:: 2.3
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage example::
|
||||
|
|
|
|||
|
|
@ -289,8 +289,10 @@ request::
|
|||
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
|
||||
|
||||
Alternatively, if you want to know the arguments needed to recreate that
|
||||
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
|
||||
function to get a dictionary with the equivalent arguments.
|
||||
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
|
||||
function to get a dictionary with the equivalent arguments:
|
||||
|
||||
.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
|
||||
|
||||
Note that to translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
|
|
|||
|
|
@ -166,8 +166,7 @@ BaseItemExporter
|
|||
By default, this method looks for a serializer :ref:`declared in the item
|
||||
field <topics-exporters-serializers>` and returns the result of applying
|
||||
that serializer to the value. If no serializer is found, it returns the
|
||||
value unchanged except for ``unicode`` values which are encoded to
|
||||
``str`` using the encoding declared in the :attr:`encoding` attribute.
|
||||
value unchanged.
|
||||
|
||||
:param field: the field being serialized. If the source :ref:`item object
|
||||
<item-types>` does not define field metadata, *field* is an empty
|
||||
|
|
@ -233,10 +232,7 @@ BaseItemExporter
|
|||
|
||||
.. attribute:: encoding
|
||||
|
||||
The encoding that will be used to encode unicode values. This only
|
||||
affects unicode values (which are always serialized to str using this
|
||||
encoding). Other value types are passed unchanged to the specific
|
||||
serialization library.
|
||||
The output character encoding.
|
||||
|
||||
.. attribute:: indent
|
||||
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ The storages backends supported out of the box are:
|
|||
* :ref:`topics-feed-storage-fs`
|
||||
* :ref:`topics-feed-storage-ftp`
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
|
||||
* :ref:`topics-feed-storage-stdout`
|
||||
|
||||
Some storage backends may be unavailable if the required external libraries are
|
||||
|
|
@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive
|
|||
mode by default. To use the active connection mode instead, set the
|
||||
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
|
||||
.. _topics-feed-storage-s3:
|
||||
|
||||
S3
|
||||
|
|
@ -194,6 +198,37 @@ You can also define a custom ACL for exported feeds using this setting:
|
|||
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
|
||||
.. _topics-feed-storage-gcs:
|
||||
|
||||
Google Cloud Storage (GCS)
|
||||
--------------------------
|
||||
|
||||
.. versionadded:: 2.3
|
||||
|
||||
The feeds are stored on `Google Cloud Storage`_.
|
||||
|
||||
* URI scheme: ``gs``
|
||||
* Example URIs:
|
||||
|
||||
* ``gs://mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `google-cloud-storage`_.
|
||||
|
||||
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
|
||||
|
||||
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
|
||||
|
||||
* :setting:`FEED_STORAGE_GCS_ACL`
|
||||
* :setting:`GCS_PROJECT_ID`
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
|
||||
|
||||
|
||||
.. _topics-feed-storage-stdout:
|
||||
|
||||
Standard output
|
||||
|
|
@ -206,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process.
|
|||
* Required external libraries: none
|
||||
|
||||
|
||||
.. _delayed-file-delivery:
|
||||
|
||||
Delayed file delivery
|
||||
---------------------
|
||||
|
||||
As indicated above, some of the described storage backends use delayed file
|
||||
delivery.
|
||||
|
||||
These storage backends do not upload items to the feed URI as those items are
|
||||
scraped. Instead, Scrapy writes items into a temporary local file, and only
|
||||
once all the file contents have been written (i.e. at the end of the crawl) is
|
||||
that file uploaded to the feed URI.
|
||||
|
||||
If you want item delivery to start earlier when using one of these storage
|
||||
backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
|
||||
in multiple files, with the specified maximum item count per file. That way, as
|
||||
soon as a file reaches the maximum item count, that file is delivered to the
|
||||
feed URI, allowing item delivery to start way before the end of the crawl.
|
||||
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
|
|
@ -220,6 +275,7 @@ These are the settings used for configuring the feed exports:
|
|||
* :setting:`FEED_STORAGE_FTP_ACTIVE`
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
* :setting:`FEED_EXPORTERS`
|
||||
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
|
||||
.. currentmodule:: scrapy.extensions.feedexport
|
||||
|
||||
|
|
@ -271,6 +327,7 @@ as a fallback value if that key is not provided for a specific feed definition.
|
|||
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
|
||||
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
|
||||
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
|
||||
* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -416,7 +473,53 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
'csv': None,
|
||||
}
|
||||
|
||||
|
||||
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
|
||||
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT
|
||||
-----------------------------
|
||||
|
||||
Default: ``0``
|
||||
|
||||
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
|
||||
storing up to the specified number of items in each output file.
|
||||
|
||||
When generating multiple output files, you must use at least one of the following
|
||||
placeholders in the feed URI to indicate how the different output file names are
|
||||
generated:
|
||||
|
||||
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
|
||||
(e.g. ``2020-03-28T14-45-08.237134``)
|
||||
|
||||
* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
|
||||
|
||||
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
|
||||
alter the number format. For example, to make the batch ID a 5-digit
|
||||
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
|
||||
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
|
||||
|
||||
For instance, if your settings include::
|
||||
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT = 100
|
||||
|
||||
And your :command:`crawl` command line is::
|
||||
|
||||
scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
|
||||
|
||||
The command line above can generate a directory tree like::
|
||||
|
||||
->projectname
|
||||
-->dirname
|
||||
--->1-filename2020-03-28T14-45-08.237134.json
|
||||
--->2-filename2020-03-28T14-45-09.148903.json
|
||||
--->3-filename2020-03-28T14-45-10.046092.json
|
||||
|
||||
Where the first and second files contain exactly 100 items. The last one contains
|
||||
100 items or fewer.
|
||||
|
||||
|
||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
||||
|
|
|
|||
|
|
@ -20,13 +20,17 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
|
|||
for extending and overriding different field parsing rules, either by spider,
|
||||
or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
|
||||
|
||||
.. note:: Item Loaders are an extension of the itemloaders_ library that make it
|
||||
easier to work with Scrapy by adding support for
|
||||
:ref:`responses <topics-request-response>`.
|
||||
|
||||
Using Item Loaders to populate items
|
||||
====================================
|
||||
|
||||
To use an Item Loader, you must first instantiate it. You can either
|
||||
instantiate it with an :ref:`item object <topics-items>` or without one, in which
|
||||
case an instance of :class:`~scrapy.item.Item` is automatically created in the
|
||||
Item Loader ``__init__`` method using the :class:`~scrapy.item.Item` subclass
|
||||
case an :ref:`item object <topics-items>` is automatically created in the
|
||||
Item Loader ``__init__`` method using the :ref:`item <topics-items>` class
|
||||
specified in the :attr:`ItemLoader.default_item_class` attribute.
|
||||
|
||||
Then, you start collecting values into the Item Loader, typically using
|
||||
|
|
@ -76,6 +80,31 @@ called which actually returns the item populated with the data
|
|||
previously extracted and collected with the :meth:`~ItemLoader.add_xpath`,
|
||||
:meth:`~ItemLoader.add_css`, and :meth:`~ItemLoader.add_value` calls.
|
||||
|
||||
|
||||
.. _topics-loaders-dataclass:
|
||||
|
||||
Working with dataclass items
|
||||
============================
|
||||
|
||||
By default, :ref:`dataclass items <dataclass-items>` require all fields to be
|
||||
passed when created. This could be an issue when using dataclass items with
|
||||
item loaders: unless a pre-populated item is passed to the loader, fields
|
||||
will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
|
||||
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
|
||||
|
||||
One approach to overcome this is to define items using the
|
||||
:func:`~dataclasses.field` function, with a ``default`` argument::
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
@dataclass
|
||||
class InventoryItem:
|
||||
name: Optional[str] = field(default=None)
|
||||
price: Optional[float] = field(default=None)
|
||||
stock: Optional[int] = field(default=None)
|
||||
|
||||
|
||||
.. _topics-loaders-processors:
|
||||
|
||||
Input and Output processors
|
||||
|
|
@ -148,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input
|
|||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
||||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
Last, but not least, itemloaders_ comes with some :ref:`commonly used
|
||||
processors <itemloaders:built-in-processors>` built-in for convenience.
|
||||
|
||||
|
||||
Declaring Item Loaders
|
||||
|
|
@ -157,17 +186,17 @@ Declaring Item Loaders
|
|||
|
||||
Item Loaders are declared using a class definition syntax. Here is an example::
|
||||
|
||||
from itemloaders.processors import TakeFirst, MapCompose, Join
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.processors import TakeFirst, MapCompose, Join
|
||||
|
||||
class ProductLoader(ItemLoader):
|
||||
|
||||
default_output_processor = TakeFirst()
|
||||
|
||||
name_in = MapCompose(unicode.title)
|
||||
name_in = MapCompose(str.title)
|
||||
name_out = Join()
|
||||
|
||||
price_in = MapCompose(unicode.strip)
|
||||
price_in = MapCompose(str.strip)
|
||||
|
||||
# ...
|
||||
|
||||
|
|
@ -189,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
|
|||
metadata. Here is an example::
|
||||
|
||||
import scrapy
|
||||
from scrapy.loader.processors import Join, MapCompose, TakeFirst
|
||||
from itemloaders.processors import Join, MapCompose, TakeFirst
|
||||
from w3lib.html import remove_tags
|
||||
|
||||
def filter_price(value):
|
||||
|
|
@ -208,10 +237,10 @@ metadata. Here is an example::
|
|||
|
||||
>>> from scrapy.loader import ItemLoader
|
||||
>>> il = ItemLoader(item=Product())
|
||||
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
|
||||
>>> il.add_value('price', [u'€', u'<span>1000</span>'])
|
||||
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
|
||||
>>> il.add_value('price', ['€', '<span>1000</span>'])
|
||||
>>> il.load_item()
|
||||
{'name': u'Welcome to my website', 'price': u'1000'}
|
||||
{'name': 'Welcome to my website', 'price': '1000'}
|
||||
|
||||
The precedence order, for both input and output processors, is as follows:
|
||||
|
||||
|
|
@ -270,250 +299,9 @@ There are several ways to modify Item Loader context values:
|
|||
ItemLoader objects
|
||||
==================
|
||||
|
||||
.. class:: ItemLoader([item, selector, response], **kwargs)
|
||||
|
||||
Return a new Item Loader for populating the given :ref:`item object
|
||||
<topics-items>`. If no item object is given, one is instantiated
|
||||
automatically using the class in :attr:`default_item_class`.
|
||||
|
||||
When instantiated with a ``selector`` or a ``response`` parameters
|
||||
the :class:`ItemLoader` class provides convenient mechanisms for extracting
|
||||
data from web pages using :ref:`selectors <topics-selectors>`.
|
||||
|
||||
:param item: The item instance to populate using subsequent calls to
|
||||
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
|
||||
or :meth:`~ItemLoader.add_value`.
|
||||
:type item: :ref:`item object <topics-items>`
|
||||
|
||||
:param selector: The selector to extract data from, when using the
|
||||
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
|
||||
(resp. :meth:`replace_css`) method.
|
||||
:type selector: :class:`~scrapy.selector.Selector` object
|
||||
|
||||
:param response: The response used to construct the selector using the
|
||||
:attr:`default_selector_class`, unless the selector argument is given,
|
||||
in which case this argument is ignored.
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
The item, selector, response and the remaining keyword arguments are
|
||||
assigned to the Loader context (accessible through the :attr:`context` attribute).
|
||||
|
||||
:class:`ItemLoader` instances have the following methods:
|
||||
|
||||
.. method:: get_value(value, *processors, **kwargs)
|
||||
|
||||
Process the given ``value`` by the given ``processors`` and keyword
|
||||
arguments.
|
||||
|
||||
Available keyword arguments:
|
||||
|
||||
:param re: a regular expression to use for extracting data from the
|
||||
given value using :meth:`~scrapy.utils.misc.extract_regex` method,
|
||||
applied before processors
|
||||
:type re: str or compiled regex
|
||||
|
||||
Examples:
|
||||
|
||||
>>> from scrapy.loader.processors import TakeFirst
|
||||
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
|
||||
'FOO`
|
||||
|
||||
.. method:: add_value(field_name, value, *processors, **kwargs)
|
||||
|
||||
Process and then add the given ``value`` for the given field.
|
||||
|
||||
The value is first passed through :meth:`get_value` by giving the
|
||||
``processors`` and ``kwargs``, and then passed through the
|
||||
:ref:`field input processor <topics-loaders-processors>` and its result
|
||||
appended to the data collected for that field. If the field already
|
||||
contains collected data, the new data is added.
|
||||
|
||||
The given ``field_name`` can be ``None``, in which case values for
|
||||
multiple fields may be added. And the processed value should be a dict
|
||||
with field_name mapped to values.
|
||||
|
||||
Examples::
|
||||
|
||||
loader.add_value('name', u'Color TV')
|
||||
loader.add_value('colours', [u'white', u'blue'])
|
||||
loader.add_value('length', u'100')
|
||||
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
|
||||
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
|
||||
|
||||
.. method:: replace_value(field_name, value, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_value` but replaces the collected data with the
|
||||
new value instead of adding it.
|
||||
.. method:: get_xpath(xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of unicode strings from the
|
||||
selector associated with this :class:`ItemLoader`.
|
||||
|
||||
:param xpath: the XPath to extract data from
|
||||
:type xpath: str
|
||||
|
||||
:param re: a regular expression to use for extracting data from the
|
||||
selected XPath region
|
||||
:type re: str or compiled regex
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.get_xpath('//p[@class="product-name"]')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
|
||||
|
||||
.. method:: add_xpath(field_name, xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of unicode strings from the
|
||||
selector associated with this :class:`ItemLoader`.
|
||||
|
||||
See :meth:`get_xpath` for ``kwargs``.
|
||||
|
||||
:param xpath: the XPath to extract data from
|
||||
:type xpath: str
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.add_xpath('name', '//p[@class="product-name"]')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
|
||||
|
||||
.. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_xpath` but replaces collected data instead of
|
||||
adding it.
|
||||
|
||||
.. method:: get_css(css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
|
||||
instead of a value, which is used to extract a list of unicode strings
|
||||
from the selector associated with this :class:`ItemLoader`.
|
||||
|
||||
:param css: the CSS selector to extract data from
|
||||
:type css: str
|
||||
|
||||
:param re: a regular expression to use for extracting data from the
|
||||
selected CSS region
|
||||
:type re: str or compiled regex
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.get_css('p.product-name')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
|
||||
|
||||
.. method:: add_css(field_name, css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
|
||||
instead of a value, which is used to extract a list of unicode strings
|
||||
from the selector associated with this :class:`ItemLoader`.
|
||||
|
||||
See :meth:`get_css` for ``kwargs``.
|
||||
|
||||
:param css: the CSS selector to extract data from
|
||||
:type css: str
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.add_css('name', 'p.product-name')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.add_css('price', 'p#price', re='the price is (.*)')
|
||||
|
||||
.. method:: replace_css(field_name, css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_css` but replaces collected data instead of
|
||||
adding it.
|
||||
|
||||
.. method:: load_item()
|
||||
|
||||
Populate the item with the data collected so far, and return it. The
|
||||
data collected is first passed through the :ref:`output processors
|
||||
<topics-loaders-processors>` to get the final value to assign to each
|
||||
item field.
|
||||
|
||||
.. method:: nested_xpath(xpath)
|
||||
|
||||
Create a nested loader with an xpath selector.
|
||||
The supplied selector is applied relative to selector associated
|
||||
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
|
||||
object <topics-items>` with the parent :class:`ItemLoader` so calls to
|
||||
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
|
||||
behave as expected.
|
||||
|
||||
.. method:: nested_css(css)
|
||||
|
||||
Create a nested loader with a css selector.
|
||||
The supplied selector is applied relative to selector associated
|
||||
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
|
||||
object <topics-items>` with the parent :class:`ItemLoader` so calls to
|
||||
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
|
||||
behave as expected.
|
||||
|
||||
.. method:: get_collected_values(field_name)
|
||||
|
||||
Return the collected values for the given field.
|
||||
|
||||
.. method:: get_output_value(field_name)
|
||||
|
||||
Return the collected values parsed using the output processor, for the
|
||||
given field. This method doesn't populate or modify the item at all.
|
||||
|
||||
.. method:: get_input_processor(field_name)
|
||||
|
||||
Return the input processor for the given field.
|
||||
|
||||
.. method:: get_output_processor(field_name)
|
||||
|
||||
Return the output processor for the given field.
|
||||
|
||||
:class:`ItemLoader` instances have the following attributes:
|
||||
|
||||
.. attribute:: item
|
||||
|
||||
The :ref:`item object <topics-items>` being parsed by this Item Loader.
|
||||
This is mostly used as a property so when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
The currently active :ref:`Context <topics-loaders-context>` of this
|
||||
Item Loader.
|
||||
|
||||
.. attribute:: default_item_class
|
||||
|
||||
An :ref:`item object <topics-items>` class or factory, used to
|
||||
instantiate items when not given in the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
The default input processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_output_processor
|
||||
|
||||
The default output processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
.. autoclass:: scrapy.loader.ItemLoader
|
||||
:members:
|
||||
:inherited-members:
|
||||
|
||||
.. _topics-loaders-nested:
|
||||
|
||||
|
|
@ -584,7 +372,7 @@ those dashes in the final product names.
|
|||
Here's how you can remove those dashes by reusing and extending the default
|
||||
Product Item Loader (``ProductLoader``)::
|
||||
|
||||
from scrapy.loader.processors import MapCompose
|
||||
from itemloaders.processors import MapCompose
|
||||
from myproject.ItemLoaders import ProductLoader
|
||||
|
||||
def strip_dashes(x):
|
||||
|
|
@ -597,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
|
|||
multiple source formats, for example XML and HTML. In the XML version you may
|
||||
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
|
||||
|
||||
from scrapy.loader.processors import MapCompose
|
||||
from itemloaders.processors import MapCompose
|
||||
from myproject.ItemLoaders import ProductLoader
|
||||
from myproject.utils.xml import remove_cdata
|
||||
|
||||
|
|
@ -617,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
|
|||
organization of your Loaders collection - that's up to you and your project's
|
||||
needs.
|
||||
|
||||
.. _topics-loaders-available-processors:
|
||||
|
||||
Available built-in processors
|
||||
=============================
|
||||
|
||||
.. module:: scrapy.loader.processors
|
||||
:synopsis: A collection of processors to use with Item Loaders
|
||||
|
||||
Even though you can use any callable function as input and output processors,
|
||||
Scrapy provides some commonly used processors, which are described below. Some
|
||||
of them, like the :class:`MapCompose` (which is typically used as input
|
||||
processor) compose the output of several functions executed in order, to
|
||||
produce the final parsed value.
|
||||
|
||||
Here is a list of all built-in processors:
|
||||
|
||||
.. class:: Identity
|
||||
|
||||
The simplest processor, which doesn't do anything. It returns the original
|
||||
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
|
||||
accept Loader contexts.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import Identity
|
||||
>>> proc = Identity()
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
['one', 'two', 'three']
|
||||
|
||||
.. class:: TakeFirst
|
||||
|
||||
Returns the first non-null/non-empty value from the values received,
|
||||
so it's typically used as an output processor to single-valued fields.
|
||||
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import TakeFirst
|
||||
>>> proc = TakeFirst()
|
||||
>>> proc(['', 'one', 'two', 'three'])
|
||||
'one'
|
||||
|
||||
.. class:: Join(separator=u' ')
|
||||
|
||||
Returns the values joined with the separator given in the ``__init__`` method, which
|
||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||
|
||||
When using the default separator, this processor is equivalent to the
|
||||
function: ``u' '.join``
|
||||
|
||||
Examples:
|
||||
|
||||
>>> from scrapy.loader.processors import Join
|
||||
>>> proc = Join()
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
'one two three'
|
||||
>>> proc = Join('<br>')
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
'one<br>two<br>three'
|
||||
|
||||
.. class:: Compose(*functions, **default_loader_context)
|
||||
|
||||
A processor which is constructed from the composition of the given
|
||||
functions. This means that each input value of this processor is passed to
|
||||
the first function, and the result of that function is passed to the second
|
||||
function, and so on, until the last function returns the output value of
|
||||
this processor.
|
||||
|
||||
By default, stop process on ``None`` value. This behaviour can be changed by
|
||||
passing keyword argument ``stop_on_none=False``.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import Compose
|
||||
>>> proc = Compose(lambda v: v[0], str.upper)
|
||||
>>> proc(['hello', 'world'])
|
||||
'HELLO'
|
||||
|
||||
Each function can optionally receive a ``loader_context`` parameter. For
|
||||
those which do, this processor will pass the currently active :ref:`Loader
|
||||
context <topics-loaders-context>` through that parameter.
|
||||
|
||||
The keyword arguments passed in the ``__init__`` method are used as the default
|
||||
Loader context values passed to each function call. However, the final
|
||||
Loader context values passed to functions are overridden with the currently
|
||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
attribute.
|
||||
|
||||
.. class:: MapCompose(*functions, **default_loader_context)
|
||||
|
||||
A processor which is constructed from the composition of the given
|
||||
functions, similar to the :class:`Compose` processor. The difference with
|
||||
this processor is the way internal results are passed among functions,
|
||||
which is as follows:
|
||||
|
||||
The input value of this processor is *iterated* and the first function is
|
||||
applied to each element. The results of these function calls (one for each element)
|
||||
are concatenated to construct a new iterable, which is then used to apply the
|
||||
second function, and so on, until the last function is applied to each
|
||||
value of the list of values collected so far. The output values of the last
|
||||
function are concatenated together to produce the output of this processor.
|
||||
|
||||
Each particular function can return a value or a list of values, which is
|
||||
flattened with the list of values returned by the same function applied to
|
||||
the other input values. The functions can also return ``None`` in which
|
||||
case the output of that function is ignored for further processing over the
|
||||
chain.
|
||||
|
||||
This processor provides a convenient way to compose functions that only
|
||||
work with single values (instead of iterables). For this reason the
|
||||
:class:`MapCompose` processor is typically used as input processor, since
|
||||
data is often extracted using the
|
||||
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
|
||||
<topics-selectors>`, which returns a list of unicode strings.
|
||||
|
||||
The example below should clarify how it works:
|
||||
|
||||
>>> def filter_world(x):
|
||||
... return None if x == 'world' else x
|
||||
...
|
||||
>>> from scrapy.loader.processors import MapCompose
|
||||
>>> proc = MapCompose(filter_world, str.upper)
|
||||
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
|
||||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
``__init__`` method keyword arguments are used as default context values. See
|
||||
:class:`Compose` processor for more info.
|
||||
|
||||
.. class:: SelectJmes(json_path)
|
||||
|
||||
Queries the value using the json path provided to the ``__init__`` method and returns the output.
|
||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||
This processor takes only one input at a time.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
|
||||
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
|
||||
>>> proc({'foo': 'bar'})
|
||||
'bar'
|
||||
>>> proc({'foo': {'bar': 'baz'}})
|
||||
{'bar': 'baz'}
|
||||
|
||||
Working with Json:
|
||||
|
||||
>>> import json
|
||||
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
|
||||
>>> proc_single_json_str('{"foo": "bar"}')
|
||||
'bar'
|
||||
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
|
||||
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
|
||||
['bar']
|
||||
.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
|
||||
.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html
|
||||
|
|
|
|||
|
|
@ -201,10 +201,12 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
|
|||
.. _s3.scality: https://s3.scality.com/
|
||||
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
||||
|
||||
.. _media-pipeline-gcs:
|
||||
|
||||
Google Cloud Storage
|
||||
---------------------
|
||||
|
||||
.. setting:: GCS_PROJECT_ID
|
||||
.. setting:: FILES_STORE_GCS_ACL
|
||||
.. setting:: IMAGES_STORE_GCS_ACL
|
||||
|
||||
|
|
@ -475,7 +477,11 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
* ``checksum`` - a `MD5 hash`_ of the image contents
|
||||
|
||||
* ``status`` - the file status indication. It can be one of the following:
|
||||
* ``status`` - the file status indication.
|
||||
|
||||
.. versionadded:: 2.2
|
||||
|
||||
It can be one of the following:
|
||||
|
||||
* ``downloaded`` - file was downloaded.
|
||||
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
||||
|
|
|
|||
|
|
@ -51,12 +51,12 @@ Request objects
|
|||
given, the dict passed in this parameter will be shallow copied.
|
||||
:type meta: dict
|
||||
|
||||
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
|
||||
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
|
||||
``body`` is not given, an empty string is stored. Regardless of the
|
||||
type of this argument, the final value stored will be a ``str`` (never
|
||||
``unicode`` or ``None``).
|
||||
:type body: str or unicode
|
||||
:param body: the request body. If a string is passed, then it's encoded as
|
||||
bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
|
||||
``body`` is not given, an empty bytes object is stored. Regardless of the
|
||||
type of this argument, the final value stored will be a bytes object
|
||||
(never a string or ``None``).
|
||||
:type body: bytes or str
|
||||
|
||||
:param headers: the headers of this request. The dict values can be strings
|
||||
(for single valued headers) or lists (for multi-valued headers). If
|
||||
|
|
@ -106,7 +106,7 @@ Request objects
|
|||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
This encoding will be used to percent-encode the URL and to convert the
|
||||
body to ``str`` (if given as ``unicode``).
|
||||
body to bytes (if given as a string).
|
||||
:type encoding: string
|
||||
|
||||
:param priority: the priority of this request (defaults to ``0``).
|
||||
|
|
@ -191,7 +191,7 @@ Request objects
|
|||
|
||||
In case of a failure to process the request, this dict can be accessed as
|
||||
``failure.request.cb_kwargs`` in the request's errback. For more information,
|
||||
see :ref:`topics-request-response-ref-accessing-callback-arguments-in-errback`.
|
||||
see :ref:`errback-cb_kwargs`.
|
||||
|
||||
.. method:: Request.copy()
|
||||
|
||||
|
|
@ -316,7 +316,7 @@ errors if needed::
|
|||
request = failure.request
|
||||
self.logger.error('TimeoutError on %s', request.url)
|
||||
|
||||
.. _topics-request-response-ref-accessing-callback-arguments-in-errback:
|
||||
.. _errback-cb_kwargs:
|
||||
|
||||
Accessing additional data in errback functions
|
||||
----------------------------------------------
|
||||
|
|
@ -721,7 +721,7 @@ Response objects
|
|||
.. attribute:: Response.body
|
||||
|
||||
The body of this Response. Keep in mind that Response.body
|
||||
is always a bytes object. If you want the unicode version use
|
||||
is always a bytes object. If you want the string version use
|
||||
:attr:`TextResponse.text` (only available in :class:`TextResponse`
|
||||
and subclasses).
|
||||
|
||||
|
|
@ -842,9 +842,9 @@ TextResponse objects
|
|||
is the same as for the :class:`Response` class and is not documented here.
|
||||
|
||||
:param encoding: is a string which contains the encoding to use for this
|
||||
response. If you create a :class:`TextResponse` object with a unicode
|
||||
response. If you create a :class:`TextResponse` object with a string as
|
||||
body, it will be encoded using this encoding (remember the body attribute
|
||||
is always a string). If ``encoding`` is ``None`` (default value), the
|
||||
is always a bytes object). If ``encoding`` is ``None`` (default value), the
|
||||
encoding will be looked up in the response headers and body instead.
|
||||
:type encoding: string
|
||||
|
||||
|
|
@ -853,7 +853,7 @@ TextResponse objects
|
|||
|
||||
.. attribute:: TextResponse.text
|
||||
|
||||
Response body, as unicode.
|
||||
Response body, as a string.
|
||||
|
||||
The same as ``response.body.decode(response.encoding)``, but the
|
||||
result is cached after the first call, so you can access
|
||||
|
|
@ -861,9 +861,11 @@ TextResponse objects
|
|||
|
||||
.. note::
|
||||
|
||||
``unicode(response.body)`` is not a correct way to convert response
|
||||
body to unicode: you would be using the system default encoding
|
||||
(typically ``ascii``) instead of the response encoding.
|
||||
``str(response.body)`` is not a correct way to convert the response
|
||||
body into a string:
|
||||
|
||||
>>> str(b'body')
|
||||
"b'body'"
|
||||
|
||||
|
||||
.. attribute:: TextResponse.encoding
|
||||
|
|
|
|||
|
|
@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
|
|||
|
||||
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
|
||||
constructed by passing either :class:`~scrapy.http.TextResponse` object or
|
||||
markup as an unicode string (in ``text`` argument).
|
||||
markup as a string (in ``text`` argument).
|
||||
|
||||
Usually there is no need to construct Scrapy selectors manually:
|
||||
``response`` object is available in Spider callbacks, so in most cases
|
||||
it is more convenient to use ``response.css()`` and ``response.xpath()``
|
||||
|
|
@ -383,7 +384,7 @@ Using selectors with regular expressions
|
|||
|
||||
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
|
||||
data using regular expressions. However, unlike using ``.xpath()`` or
|
||||
``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
|
||||
``.css()`` methods, ``.re()`` returns a list of strings. So you
|
||||
can't construct nested ``.re()`` calls.
|
||||
|
||||
Here's an example used to extract image names from the :ref:`HTML code
|
||||
|
|
@ -734,7 +735,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
|
|||
Example selecting links in list item with a "class" attribute ending with a digit:
|
||||
|
||||
>>> from scrapy import Selector
|
||||
>>> doc = u"""
|
||||
>>> doc = """
|
||||
... <div>
|
||||
... <ul>
|
||||
... <li class="item-0"><a href="link1.html">first item</a></li>
|
||||
|
|
@ -765,7 +766,7 @@ extracting text elements for example.
|
|||
Example extracting microdata (sample content taken from https://schema.org/Product)
|
||||
with groups of itemscopes and corresponding itemprops::
|
||||
|
||||
>>> doc = u"""
|
||||
>>> doc = """
|
||||
... <div itemscope itemtype="http://schema.org/Product">
|
||||
... <span itemprop="name">Kenmore White 17" Microwave</span>
|
||||
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
|
||||
|
|
@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
|
|||
sel.xpath("//h1")
|
||||
|
||||
2. Extract the text of all ``<h1>`` elements from an HTML response body,
|
||||
returning a list of unicode strings::
|
||||
returning a list of strings::
|
||||
|
||||
sel.xpath("//h1").getall() # this includes the h1 tag
|
||||
sel.xpath("//h1/text()").getall() # this excludes the h1 tag
|
||||
|
|
|
|||
|
|
@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
|
|||
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
|
||||
specific cipher that is not included in ``DEFAULT`` if a website requires it.
|
||||
|
||||
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
|
||||
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
|
||||
|
||||
|
|
@ -786,6 +786,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
|
|||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: FEED_STORAGE_GCS_ACL
|
||||
|
||||
FEED_STORAGE_GCS_ACL
|
||||
--------------------
|
||||
|
||||
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
||||
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
|
||||
|
||||
.. setting:: FTP_PASSIVE_MODE
|
||||
|
||||
FTP_PASSIVE_MODE
|
||||
|
|
@ -825,6 +833,15 @@ Default: ``"anonymous"``
|
|||
The username to use for FTP connections when there is no ``"ftp_user"``
|
||||
in ``Request`` meta.
|
||||
|
||||
.. setting:: GCS_PROJECT_ID
|
||||
|
||||
GCS_PROJECT_ID
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The Project ID that will be used when storing data on `Google Cloud Storage`_.
|
||||
|
||||
.. setting:: ITEM_PIPELINES
|
||||
|
||||
ITEM_PIPELINES
|
||||
|
|
@ -1544,3 +1561,4 @@ case to see how to enable and use them.
|
|||
.. _Amazon web services: https://aws.amazon.com/
|
||||
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
|
||||
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
|
||||
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
||||
|
|
|
|||
|
|
@ -360,9 +360,10 @@ CrawlSpider
|
|||
|
||||
This spider also exposes an overrideable method:
|
||||
|
||||
.. method:: parse_start_url(response)
|
||||
.. method:: parse_start_url(response, **kwargs)
|
||||
|
||||
This method is called for the start_urls responses. It allows to parse
|
||||
This method is called for each response produced for the URLs in
|
||||
the spider's ``start_urls`` attribute. It allows to parse
|
||||
the initial responses and must return either an
|
||||
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
|
||||
object, or an iterable containing any of them.
|
||||
|
|
@ -388,11 +389,6 @@ Crawling rules
|
|||
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
|
||||
in its ``meta`` dictionary (under the ``link_text`` key)
|
||||
|
||||
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
|
||||
callback, since the :class:`CrawlSpider` uses the ``parse`` method
|
||||
itself to implement its logic. So if you override the ``parse`` method,
|
||||
the crawl spider will no longer work.
|
||||
|
||||
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
|
||||
callback function.
|
||||
|
||||
|
|
@ -418,6 +414,11 @@ Crawling rules
|
|||
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
|
||||
instance as first parameter.
|
||||
|
||||
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
|
||||
.. versionadded:: 2.0
|
||||
The *errback* parameter.
|
||||
|
||||
|
|
@ -451,6 +452,11 @@ Let's now take a look at an example CrawlSpider with rules::
|
|||
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
|
||||
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
|
||||
item['link_text'] = response.meta['link_text']
|
||||
url = response.xpath('//td[@id="additional_data"]/@href').get()
|
||||
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
|
||||
|
||||
def parse_additional_page(self, response, item):
|
||||
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
|
||||
return item
|
||||
|
||||
|
||||
|
|
@ -544,6 +550,11 @@ XMLFeedSpider
|
|||
those results. It must return a list of results (items or requests).
|
||||
|
||||
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
|
||||
|
||||
XMLFeedSpider example
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def main():
|
|||
_contents = None
|
||||
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class QPSSpider(Spider):
|
|||
slots = 1
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(QPSSpider, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
if self.qps is not None:
|
||||
self.qps = float(self.qps)
|
||||
self.download_delay = 1 / self.qps
|
||||
|
|
|
|||
|
|
@ -40,3 +40,4 @@ flake8-ignore =
|
|||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.1.0
|
||||
2.3.0
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
|
|||
# scrapy.utils.spider.iter_spider_classes
|
||||
for module in walk_modules(module_name):
|
||||
for obj in vars(module).values():
|
||||
if inspect.isclass(obj) and \
|
||||
issubclass(obj, ScrapyCommand) and \
|
||||
obj.__module__ == module.__name__ and \
|
||||
not obj == ScrapyCommand:
|
||||
if (
|
||||
inspect.isclass(obj)
|
||||
and issubclass(obj, ScrapyCommand)
|
||||
and obj.__module__ == module.__name__
|
||||
and not obj == ScrapyCommand
|
||||
):
|
||||
yield obj
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ class ScrapyCommand:
|
|||
|
||||
class BaseRunSpiderCommand(ScrapyCommand):
|
||||
"""
|
||||
Common class used to share functionality between the crawl and runspider commands
|
||||
Common class used to share functionality between the crawl, parse and runspider commands
|
||||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
|
|
|
|||
|
|
@ -78,19 +78,19 @@ class Command(ScrapyCommand):
|
|||
elif tested_methods:
|
||||
self.crawler_process.crawl(spidercls)
|
||||
|
||||
# start checks
|
||||
if opts.list:
|
||||
for spider, methods in sorted(contract_reqs.items()):
|
||||
if not methods and not opts.verbose:
|
||||
continue
|
||||
print(spider)
|
||||
for method in sorted(methods):
|
||||
print(' * %s' % method)
|
||||
else:
|
||||
start = time.time()
|
||||
self.crawler_process.start()
|
||||
stop = time.time()
|
||||
# start checks
|
||||
if opts.list:
|
||||
for spider, methods in sorted(contract_reqs.items()):
|
||||
if not methods and not opts.verbose:
|
||||
continue
|
||||
print(spider)
|
||||
for method in sorted(methods):
|
||||
print(' * %s' % method)
|
||||
else:
|
||||
start = time.time()
|
||||
self.crawler_process.start()
|
||||
stop = time.time()
|
||||
|
||||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
|
|||
else:
|
||||
self.crawler_process.start()
|
||||
|
||||
if self.crawler_process.bootstrap_failed or \
|
||||
(hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
|
||||
if (
|
||||
self.crawler_process.bootstrap_failed
|
||||
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
|
||||
):
|
||||
self.exitcode = 1
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ class Command(ScrapyCommand):
|
|||
return "Fetch a URL using the Scrapy downloader"
|
||||
|
||||
def long_desc(self):
|
||||
return "Fetch a URL using the Scrapy downloader and print its content " \
|
||||
"to stdout. You may want to use --nolog to disable logging"
|
||||
return (
|
||||
"Fetch a URL using the Scrapy downloader and print its content"
|
||||
" to stdout. You may want to use --nolog to disable logging"
|
||||
)
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
@property
|
||||
def templates_dir(self):
|
||||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'spiders')
|
||||
return join(
|
||||
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
|
||||
'spiders'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,18 +4,16 @@ import logging
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import BaseRunSpiderCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils import display
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
class Command(BaseRunSpiderCommand):
|
||||
requires_project = True
|
||||
|
||||
spider = None
|
||||
|
|
@ -31,11 +29,9 @@ class Command(ScrapyCommand):
|
|||
return "Parse URL (using its spider) and print the results"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
|
|
@ -200,12 +196,15 @@ class Command(ScrapyCommand):
|
|||
self.add_items(depth, items)
|
||||
self.add_requests(depth, requests)
|
||||
|
||||
scraped_data = items if opts.output else []
|
||||
if depth < opts.depth:
|
||||
for req in requests:
|
||||
req.meta['_depth'] = depth + 1
|
||||
req.meta['_callback'] = req.callback
|
||||
req.callback = callback
|
||||
return requests
|
||||
scraped_data += requests
|
||||
|
||||
return scraped_data
|
||||
|
||||
# update request meta if any extra meta was passed through the --meta/-m opts.
|
||||
if opts.meta:
|
||||
|
|
@ -221,18 +220,11 @@ class Command(ScrapyCommand):
|
|||
return request
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
BaseRunSpiderCommand.process_options(self, args, opts)
|
||||
|
||||
self.process_spider_arguments(opts)
|
||||
self.process_request_meta(opts)
|
||||
self.process_request_cb_kwargs(opts)
|
||||
|
||||
def process_spider_arguments(self, opts):
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
|
||||
def process_request_meta(self, opts):
|
||||
if opts.meta:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import re
|
||||
import os
|
||||
import stat
|
||||
import string
|
||||
from importlib import import_module
|
||||
from os.path import join, exists, abspath
|
||||
from shutil import ignore_patterns, move, copy2, copystat
|
||||
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
|
|||
('${project_name}', 'middlewares.py.tmpl'),
|
||||
)
|
||||
|
||||
IGNORE = ignore_patterns('*.pyc', '.svn')
|
||||
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
|
||||
|
||||
|
||||
def _make_writable(path):
|
||||
current_permissions = os.stat(path).st_mode
|
||||
os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
|
@ -78,30 +83,10 @@ class Command(ScrapyCommand):
|
|||
self._copytree(srcname, dstname)
|
||||
else:
|
||||
copy2(srcname, dstname)
|
||||
_make_writable(dstname)
|
||||
|
||||
copystat(src, dst)
|
||||
self._set_rw_permissions(dst)
|
||||
|
||||
def _set_rw_permissions(self, path):
|
||||
"""
|
||||
Sets permissions of a directory tree to +rw and +rwx for folders.
|
||||
This is necessary if the start template files come without write
|
||||
permissions.
|
||||
"""
|
||||
mode_rw = (stat.S_IRUSR
|
||||
| stat.S_IWUSR
|
||||
| stat.S_IRGRP
|
||||
| stat.S_IROTH)
|
||||
|
||||
mode_x = (stat.S_IXUSR
|
||||
| stat.S_IXGRP
|
||||
| stat.S_IXOTH)
|
||||
|
||||
os.chmod(path, mode_rw | mode_x)
|
||||
for root, dirs, files in os.walk(path):
|
||||
for dir in dirs:
|
||||
os.chmod(join(root, dir), mode_rw | mode_x)
|
||||
for file in files:
|
||||
os.chmod(join(root, file), mode_rw)
|
||||
_make_writable(dst)
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) not in (1, 2):
|
||||
|
|
@ -137,6 +122,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
@property
|
||||
def templates_dir(self):
|
||||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'project')
|
||||
return join(
|
||||
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
|
||||
'project'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,11 +8,10 @@ class Command(fetch.Command):
|
|||
return "Open URL in browser, as seen by Scrapy"
|
||||
|
||||
def long_desc(self):
|
||||
return "Fetch a URL using the Scrapy downloader and show its " \
|
||||
"contents in a browser"
|
||||
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
|
||||
|
||||
def add_options(self, parser):
|
||||
super(Command, self).add_options(parser)
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ class ReturnsContract(Contract):
|
|||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ReturnsContract, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
if len(self.args) not in [1, 2, 3]:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
"""
|
||||
|
||||
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
|
||||
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self._ssl_method = method
|
||||
self.tls_verbose_logging = tls_verbose_logging
|
||||
if tls_ciphers:
|
||||
|
|
@ -45,7 +45,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
|
||||
#
|
||||
# * getattr() for `_ssl_method` attribute for context factories
|
||||
# not calling super(..., self).__init__
|
||||
# not calling super().__init__
|
||||
return CertificateOptions(
|
||||
verify=False,
|
||||
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
|
||||
|
|
|
|||
|
|
@ -126,7 +126,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
|
||||
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
|
||||
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
|
||||
super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
|
||||
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
|
||||
self._tunnelReadyDeferred = defer.Deferred()
|
||||
self._tunneledHost = host
|
||||
self._tunneledPort = port
|
||||
|
|
@ -178,7 +178,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
|
||||
def connect(self, protocolFactory):
|
||||
self._protocolFactory = protocolFactory
|
||||
connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
|
||||
connectDeferred = super().connect(protocolFactory)
|
||||
connectDeferred.addCallback(self.requestTunnel)
|
||||
connectDeferred.addErrback(self.connectFailed)
|
||||
return self._tunnelReadyDeferred
|
||||
|
|
@ -215,7 +215,7 @@ class TunnelingAgent(Agent):
|
|||
|
||||
def __init__(self, reactor, proxyConf, contextFactory=None,
|
||||
connectTimeout=None, bindAddress=None, pool=None):
|
||||
super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
|
||||
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
|
||||
self._proxyConf = proxyConf
|
||||
self._contextFactory = contextFactory
|
||||
|
||||
|
|
@ -235,7 +235,7 @@ class TunnelingAgent(Agent):
|
|||
# otherwise, same remote host connection request could reuse
|
||||
# a cached tunneled connection to a different proxy
|
||||
key = key + self._proxyConf
|
||||
return super(TunnelingAgent, self)._requestWithEndpoint(
|
||||
return super()._requestWithEndpoint(
|
||||
key=key,
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
|
|
@ -249,7 +249,7 @@ class TunnelingAgent(Agent):
|
|||
class ScrapyProxyAgent(Agent):
|
||||
|
||||
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
|
||||
super(ScrapyProxyAgent, self).__init__(
|
||||
super().__init__(
|
||||
reactor=reactor,
|
||||
connectTimeout=connectTimeout,
|
||||
bindAddress=bindAddress,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
"""
|
||||
|
||||
def __init__(self, hostname, ctx, verbose_logging=False):
|
||||
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
|
||||
super().__init__(hostname, ctx)
|
||||
self.verbose_logging = verbose_logging
|
||||
|
||||
def _identityVerifyingInfoCallback(self, connection, where, ret):
|
||||
|
|
|
|||
|
|
@ -141,10 +141,12 @@ class ExecutionEngine:
|
|||
|
||||
def _needs_backout(self, spider):
|
||||
slot = self.slot
|
||||
return not self.running \
|
||||
or slot.closing \
|
||||
or self.downloader.needs_backout() \
|
||||
return (
|
||||
not self.running
|
||||
or slot.closing
|
||||
or self.downloader.needs_backout()
|
||||
or self.scraper.slot.needs_backout()
|
||||
)
|
||||
|
||||
def _next_request_from_scheduler(self, spider):
|
||||
slot = self.slot
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ class Scraper:
|
|||
def call_spider(self, result, request, spider):
|
||||
result.request = request
|
||||
dfd = defer_result(result)
|
||||
callback = request.callback or spider.parse
|
||||
callback = request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
dfd.addCallbacks(callback=callback,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
|
||||
|
||||
def _add_middleware(self, mw):
|
||||
super(SpiderMiddlewareManager, self)._add_middleware(mw)
|
||||
super()._add_middleware(mw)
|
||||
if hasattr(mw, 'process_spider_input'):
|
||||
self.methods['process_spider_input'].append(mw.process_spider_input)
|
||||
if hasattr(mw, 'process_start_requests'):
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
"""
|
||||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
super(CrawlerProcess, self).__init__(settings)
|
||||
super().__init__(settings)
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
|
|
|
|||
|
|
@ -33,10 +33,8 @@ class BaseRedirectMiddleware:
|
|||
if ttl and redirects <= self.max_redirect_times:
|
||||
redirected.meta['redirect_times'] = redirects
|
||||
redirected.meta['redirect_ttl'] = ttl - 1
|
||||
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
|
||||
[request.url]
|
||||
redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
|
||||
[reason]
|
||||
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
|
||||
redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
|
||||
redirected.dont_filter = request.dont_filter
|
||||
redirected.priority = request.priority + self.priority_adjust
|
||||
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
|
||||
|
|
@ -94,13 +92,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
|
|||
enabled_setting = 'METAREFRESH_ENABLED'
|
||||
|
||||
def __init__(self, settings):
|
||||
super(MetaRefreshMiddleware, self).__init__(settings)
|
||||
super().__init__(settings)
|
||||
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
|
||||
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
|
||||
not isinstance(response, HtmlResponse):
|
||||
if (
|
||||
request.meta.get('dont_redirect', False)
|
||||
or request.method == 'HEAD'
|
||||
or not isinstance(response, HtmlResponse)
|
||||
):
|
||||
return response
|
||||
|
||||
interval, url = get_meta_refresh(response,
|
||||
|
|
|
|||
|
|
@ -60,8 +60,10 @@ class RetryMiddleware:
|
|||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
|
||||
and not request.meta.get('dont_retry', False):
|
||||
if (
|
||||
isinstance(exception, self.EXCEPTIONS_TO_RETRY)
|
||||
and not request.meta.get('dont_retry', False)
|
||||
):
|
||||
return self._retry(request, exception, spider)
|
||||
|
||||
def _retry(self, request, reason, spider):
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class CloseSpider(Exception):
|
|||
"""Raise this from callbacks to request the spider to be closed"""
|
||||
|
||||
def __init__(self, reason='cancelled'):
|
||||
super(CloseSpider, self).__init__()
|
||||
super().__init__()
|
||||
self.reason = reason
|
||||
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ class UsageError(Exception):
|
|||
|
||||
def __init__(self, *a, **kw):
|
||||
self.print_help = kw.pop('print_help', True)
|
||||
super(UsageError, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
|
||||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
|
|
|
|||
|
|
@ -256,12 +256,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def _write_headers_and_set_fields_to_export(self, item):
|
||||
if self.include_headers_line:
|
||||
if not self.fields_to_export:
|
||||
if isinstance(item, dict):
|
||||
# for dicts try using fields of the first item
|
||||
self.fields_to_export = list(item.keys())
|
||||
else:
|
||||
# use fields declared in Item
|
||||
self.fields_to_export = list(item.fields.keys())
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
if isinstance(self.fields_to_export, Mapping):
|
||||
fields = self.fields_to_export.values()
|
||||
else:
|
||||
|
|
@ -322,7 +318,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
|
||||
def _configure(self, options, dont_fail=False):
|
||||
self.binary = options.pop('binary', True)
|
||||
super(PythonItemExporter, self)._configure(options, dont_fail)
|
||||
super()._configure(options, dont_fail)
|
||||
if self.binary:
|
||||
warnings.warn(
|
||||
"PythonItemExporter will drop support for binary export in the future",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
|
|||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
|
|
@ -153,6 +154,32 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
key.close()
|
||||
|
||||
|
||||
class GCSFeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri, project_id, acl):
|
||||
self.project_id = project_id
|
||||
self.acl = acl
|
||||
u = urlparse(uri)
|
||||
self.bucket_name = u.hostname
|
||||
self.blob_name = u.path[1:] # remove first "/"
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return cls(
|
||||
uri,
|
||||
crawler.settings['GCS_PROJECT_ID'],
|
||||
crawler.settings['FEED_STORAGE_GCS_ACL'] or None
|
||||
)
|
||||
|
||||
def _store_in_thread(self, file):
|
||||
file.seek(0)
|
||||
from google.cloud.storage import Client
|
||||
client = Client(project=self.project_id)
|
||||
bucket = client.get_bucket(self.bucket_name)
|
||||
blob = bucket.blob(self.blob_name)
|
||||
blob.upload_from_file(file, predefined_acl=self.acl)
|
||||
|
||||
|
||||
class FTPFeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri, use_active_mode=False):
|
||||
|
|
@ -180,14 +207,16 @@ class FTPFeedStorage(BlockingFeedStorage):
|
|||
|
||||
|
||||
class _FeedSlot:
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty):
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
|
||||
self.file = file
|
||||
self.exporter = exporter
|
||||
self.storage = storage
|
||||
# feed params
|
||||
self.uri = uri
|
||||
self.batch_id = batch_id
|
||||
self.format = format
|
||||
self.store_empty = store_empty
|
||||
self.uri_template = uri_template
|
||||
self.uri = uri
|
||||
# flags
|
||||
self.itemcount = 0
|
||||
self._exporting = False
|
||||
|
|
@ -244,54 +273,112 @@ class FeedExporter:
|
|||
for uri, feed in self.feeds.items():
|
||||
if not self._storage_supported(uri):
|
||||
raise NotConfigured
|
||||
if not self._settings_are_valid():
|
||||
raise NotConfigured
|
||||
if not self._exporter_supported(feed['format']):
|
||||
raise NotConfigured
|
||||
|
||||
def open_spider(self, spider):
|
||||
for uri, feed in self.feeds.items():
|
||||
uri = uri % self._get_uri_params(spider, feed['uri_params'])
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed['format'],
|
||||
fields_to_export=feed['fields'],
|
||||
encoding=feed['encoding'],
|
||||
indent=feed['indent'],
|
||||
)
|
||||
slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
|
||||
self.slots.append(slot)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
uri_params = self._get_uri_params(spider, feed['uri_params'])
|
||||
self.slots.append(self._start_new_batch(
|
||||
batch_id=1,
|
||||
uri=uri % uri_params,
|
||||
feed=feed,
|
||||
spider=spider,
|
||||
uri_template=uri,
|
||||
))
|
||||
|
||||
def close_spider(self, spider):
|
||||
deferred_list = []
|
||||
for slot in self.slots:
|
||||
if not slot.itemcount and not slot.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
deferred_list.append(d)
|
||||
continue
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
d.addCallback(lambda _: logger.info(logfmt % "Stored", log_args,
|
||||
extra={'spider': spider}))
|
||||
d.addErrback(lambda f: logger.error(logfmt % "Error storing", log_args,
|
||||
exc_info=failure_to_exc_info(f),
|
||||
extra={'spider': spider}))
|
||||
d = self._close_slot(slot, spider)
|
||||
deferred_list.append(d)
|
||||
return defer.DeferredList(deferred_list) if deferred_list else None
|
||||
|
||||
def _close_slot(self, slot, spider):
|
||||
if not slot.itemcount and not slot.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
|
||||
# Use `largs=log_args` to copy log_args into function's scope
|
||||
# instead of using `log_args` from the outer scope
|
||||
d.addCallback(
|
||||
lambda _, largs=log_args: logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
)
|
||||
)
|
||||
d.addErrback(
|
||||
lambda f, largs=log_args: logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
)
|
||||
return d
|
||||
|
||||
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
|
||||
"""
|
||||
Redirect the output data stream to a new file.
|
||||
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
|
||||
:param batch_id: sequence number of current batch
|
||||
:param uri: uri of the new batch to start
|
||||
:param feed: dict with parameters of feed
|
||||
:param spider: user spider
|
||||
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
|
||||
"""
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed['format'],
|
||||
fields_to_export=feed['fields'],
|
||||
encoding=feed['encoding'],
|
||||
indent=feed['indent'],
|
||||
)
|
||||
slot = _FeedSlot(
|
||||
file=file,
|
||||
exporter=exporter,
|
||||
storage=storage,
|
||||
uri=uri,
|
||||
format=feed['format'],
|
||||
store_empty=feed['store_empty'],
|
||||
batch_id=batch_id,
|
||||
uri_template=uri_template,
|
||||
)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
return slot
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
slots = []
|
||||
for slot in self.slots:
|
||||
slot.start_exporting()
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
|
||||
if (
|
||||
self.feeds[slot.uri_template]['batch_item_count']
|
||||
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
|
||||
):
|
||||
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
|
||||
self._close_slot(slot, spider)
|
||||
slots.append(self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=slot.uri_template % uri_params,
|
||||
feed=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
uri_template=slot.uri_template,
|
||||
))
|
||||
else:
|
||||
slots.append(slot)
|
||||
self.slots = slots
|
||||
|
||||
def _load_components(self, setting_prefix):
|
||||
conf = without_none_values(self.settings.getwithbase(setting_prefix))
|
||||
|
|
@ -308,6 +395,22 @@ class FeedExporter:
|
|||
return True
|
||||
logger.error("Unknown feed format: %(format)s", {'format': format})
|
||||
|
||||
def _settings_are_valid(self):
|
||||
"""
|
||||
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
|
||||
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
|
||||
"""
|
||||
for uri_template, values in self.feeds.items():
|
||||
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
|
||||
logger.error(
|
||||
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
|
||||
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
|
||||
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
|
||||
''.format(uri_template)
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _storage_supported(self, uri):
|
||||
scheme = urlparse(uri).scheme
|
||||
if scheme in self.storages:
|
||||
|
|
@ -333,12 +436,14 @@ class FeedExporter:
|
|||
def _get_storage(self, uri):
|
||||
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
|
||||
|
||||
def _get_uri_params(self, spider, uri_params):
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['time'] = ts
|
||||
utc_now = datetime.utcnow()
|
||||
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['batch_time'] = utc_now.isoformat().replace(':', '-')
|
||||
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
|
|
|
|||
|
|
@ -81,8 +81,10 @@ class MemoryUsage:
|
|||
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
subj = "%s terminated: memory usage exceeded %dM at %s" % \
|
||||
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
subj = (
|
||||
"%s terminated: memory usage exceeded %dM at %s"
|
||||
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
)
|
||||
self._send_report(self.notify_mails, subj)
|
||||
self.crawler.stats.set_value('memusage/limit_notified', 1)
|
||||
|
||||
|
|
@ -102,8 +104,10 @@ class MemoryUsage:
|
|||
logger.warning("Memory usage reached %(memusage)dM",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
subj = "%s warning: memory usage reached %dM at %s" % \
|
||||
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
subj = (
|
||||
"%s warning: memory usage reached %dM at %s"
|
||||
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
)
|
||||
self._send_report(self.notify_mails, subj)
|
||||
self.crawler.stats.set_value('memusage/warning_notified', 1)
|
||||
self.warned = True
|
||||
|
|
|
|||
|
|
@ -186,9 +186,6 @@ class WrappedResponse:
|
|||
def info(self):
|
||||
return self
|
||||
|
||||
# python3 cookiejars calls get_all
|
||||
def get_all(self, name, default=None):
|
||||
return [to_unicode(v, errors='replace')
|
||||
for v in self.response.headers.getlist(name)]
|
||||
# python2 cookiejars calls getheaders
|
||||
getheaders = get_all
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ class Headers(CaselessDict):
|
|||
|
||||
def __init__(self, seq=None, encoding='utf-8'):
|
||||
self.encoding = encoding
|
||||
super(Headers, self).__init__(seq)
|
||||
super().__init__(seq)
|
||||
|
||||
def normkey(self, key):
|
||||
"""Normalize key to bytes"""
|
||||
|
|
@ -37,19 +37,19 @@ class Headers(CaselessDict):
|
|||
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return super(Headers, self).__getitem__(key)[-1]
|
||||
return super().__getitem__(key)[-1]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def get(self, key, def_val=None):
|
||||
try:
|
||||
return super(Headers, self).get(key, def_val)[-1]
|
||||
return super().get(key, def_val)[-1]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def getlist(self, key, def_val=None):
|
||||
try:
|
||||
return super(Headers, self).__getitem__(key)
|
||||
return super().__getitem__(key)
|
||||
except KeyError:
|
||||
if def_val is not None:
|
||||
return self.normvalue(def_val)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ class FormRequest(Request):
|
|||
if formdata and kwargs.get('method') is None:
|
||||
kwargs['method'] = 'POST'
|
||||
|
||||
super(FormRequest, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
if formdata:
|
||||
items = formdata.items() if isinstance(formdata, dict) else formdata
|
||||
|
|
@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
|
|||
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
|
||||
namespaces={
|
||||
"re": "http://exslt.org/regular-expressions"})
|
||||
values = [(k, u'' if v is None else v)
|
||||
values = [(k, '' if v is None else v)
|
||||
for k, v in (_value(e) for e in inputs)
|
||||
if k and k not in formdata_keys]
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ def _select_value(ele, n, v):
|
|||
# This is a workround to bug in lxml fixed 2.3.1
|
||||
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
|
||||
selected_options = ele.xpath('.//option[@selected]')
|
||||
v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
|
||||
v = [(o.get('value') or o.text or '').strip() for o in selected_options]
|
||||
return n, v
|
||||
|
||||
|
||||
|
|
@ -205,8 +205,7 @@ def _get_clickable(clickdata, form):
|
|||
|
||||
# We didn't find it, so now we build an XPath expression out of the other
|
||||
# arguments, because they can be used as such
|
||||
xpath = u'.//*' + \
|
||||
u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
|
||||
xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
|
||||
el = form.xpath(xpath)
|
||||
if len(el) == 1:
|
||||
return (el[0].get('name'), el[0].get('value') or '')
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class JsonRequest(Request):
|
|||
if 'method' not in kwargs:
|
||||
kwargs['method'] = 'POST'
|
||||
|
||||
super(JsonRequest, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.headers.setdefault('Content-Type', 'application/json')
|
||||
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ class JsonRequest(Request):
|
|||
elif not body_passed and data_passed:
|
||||
kwargs['body'] = self._dumps(data)
|
||||
|
||||
return super(JsonRequest, self).replace(*args, **kwargs)
|
||||
return super().replace(*args, **kwargs)
|
||||
|
||||
def _dumps(self, data):
|
||||
"""Convert to JSON """
|
||||
|
|
|
|||
|
|
@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
|
|||
if encoding is not None:
|
||||
kwargs['encoding'] = encoding
|
||||
|
||||
super(XmlRpcRequest, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.headers.setdefault('Content-Type', 'text/xml')
|
||||
|
|
|
|||
|
|
@ -35,13 +35,13 @@ class TextResponse(Response):
|
|||
self._cached_benc = None
|
||||
self._cached_ubody = None
|
||||
self._cached_selector = None
|
||||
super(TextResponse, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, str):
|
||||
self._url = to_unicode(url, self.encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_url(url)
|
||||
super()._set_url(url)
|
||||
|
||||
def _set_body(self, body):
|
||||
self._body = b'' # used by encoding detection
|
||||
|
|
@ -51,7 +51,7 @@ class TextResponse(Response):
|
|||
type(self).__name__)
|
||||
self._body = body.encode(self._encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_body(body)
|
||||
super()._set_body(body)
|
||||
|
||||
def replace(self, *args, **kwargs):
|
||||
kwargs.setdefault('encoding', self.encoding)
|
||||
|
|
@ -62,8 +62,11 @@ class TextResponse(Response):
|
|||
return self._declared_encoding() or self._body_inferred_encoding()
|
||||
|
||||
def _declared_encoding(self):
|
||||
return self._encoding or self._headers_encoding() \
|
||||
return (
|
||||
self._encoding
|
||||
or self._headers_encoding()
|
||||
or self._body_declared_encoding()
|
||||
)
|
||||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
|
|
@ -74,6 +77,8 @@ class TextResponse(Response):
|
|||
|
||||
def json(self):
|
||||
"""
|
||||
.. versionadded:: 2.2
|
||||
|
||||
Deserialize a JSON document to a Python object.
|
||||
"""
|
||||
if self._cached_decoded_json is _NONE:
|
||||
|
|
@ -161,7 +166,7 @@ class TextResponse(Response):
|
|||
elif isinstance(url, parsel.SelectorList):
|
||||
raise ValueError("SelectorList is not supported")
|
||||
encoding = self.encoding if encoding is None else encoding
|
||||
return super(TextResponse, self).follow(
|
||||
return super().follow(
|
||||
url=url,
|
||||
callback=callback,
|
||||
method=method,
|
||||
|
|
@ -221,7 +226,7 @@ class TextResponse(Response):
|
|||
for sel in selectors:
|
||||
with suppress(_InvalidSelector):
|
||||
urls.append(_url_from_selector(sel))
|
||||
return super(TextResponse, self).follow_all(
|
||||
return super().follow_all(
|
||||
urls=urls,
|
||||
callback=callback,
|
||||
method=method,
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
|||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(BaseItem, cls).__new__(cls, *args, **kwargs)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
|
|
@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
|
|||
def __new__(mcs, class_name, bases, attrs):
|
||||
classcell = attrs.pop('__classcell__', None)
|
||||
new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
|
||||
_class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
|
||||
_class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
|
||||
|
||||
fields = getattr(_class, 'fields', {})
|
||||
new_attrs = {}
|
||||
|
|
@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
|
|||
new_attrs['_class'] = _class
|
||||
if classcell is not None:
|
||||
new_attrs['__classcell__'] = classcell
|
||||
return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
|
||||
return super().__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
class DictItem(MutableMapping, BaseItem):
|
||||
|
|
@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(DictItem, cls).__new__(cls, *args, **kwargs)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
|
@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __setattr__(self, name, value):
|
||||
if not name.startswith('_'):
|
||||
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
|
||||
super(DictItem, self).__setattr__(name, value)
|
||||
super().__setattr__(name, value)
|
||||
|
||||
def __len__(self):
|
||||
return len(self._values)
|
||||
|
|
|
|||
|
|
@ -21,12 +21,18 @@ class Link:
|
|||
self.nofollow = nofollow
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.url == other.url and self.text == other.text and \
|
||||
self.fragment == other.fragment and self.nofollow == other.nofollow
|
||||
return (
|
||||
self.url == other.url
|
||||
and self.text == other.text
|
||||
and self.fragment == other.fragment
|
||||
and self.nofollow == other.nofollow
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
|
||||
(self.url, self.text, self.fragment, self.nofollow)
|
||||
return (
|
||||
'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
|
||||
% (self.url, self.text, self.fragment, self.nofollow)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ class FilteringLinkExtractor:
|
|||
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
|
||||
'please use scrapy.linkextractors.LinkExtractor instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(FilteringLinkExtractor, cls).__new__(cls)
|
||||
return super().__new__(cls)
|
||||
|
||||
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
|
||||
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
"""
|
||||
HTMLParser-based link extractor
|
||||
"""
|
||||
import warnings
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
class HtmlParserLinkExtractor(HTMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False,
|
||||
strip=True):
|
||||
HTMLParser.__init__(self)
|
||||
|
||||
warnings.warn(
|
||||
"HtmlParserLinkExtractor is deprecated and will be removed in "
|
||||
"future releases. Please use scrapy.linkextractors.LinkExtractor",
|
||||
ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding):
|
||||
self.reset()
|
||||
self.feed(response_text)
|
||||
self.close()
|
||||
|
||||
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
|
||||
|
||||
ret = []
|
||||
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in links:
|
||||
if isinstance(link.url, str):
|
||||
link.url = link.url.encode(response_encoding)
|
||||
try:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
except ValueError:
|
||||
continue
|
||||
link.url = safe_url_string(link.url, response_encoding)
|
||||
link.text = link.text.decode(response_encoding)
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
return self._extract_links(response.body, response.url, response.encoding)
|
||||
|
||||
def reset(self):
|
||||
HTMLParser.reset(self)
|
||||
|
||||
self.base_url = None
|
||||
self.current_link = None
|
||||
self.links = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == 'base':
|
||||
self.base_url = dict(attrs).get('href')
|
||||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if self.strip:
|
||||
value = strip_html5_whitespace(value)
|
||||
url = self.process_attr(value)
|
||||
link = Link(url=url)
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.scan_tag(tag):
|
||||
self.current_link = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.current_link:
|
||||
self.current_link.text = self.current_link.text + data
|
||||
|
||||
def matches(self, url):
|
||||
"""This extractor matches with any url, since
|
||||
it doesn't contain any patterns"""
|
||||
return True
|
||||
|
|
@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
|
|||
url = safe_url_string(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
link = Link(url, _collect_string_content(el) or '',
|
||||
nofollow=rel_has_nofollow(el.get('rel')))
|
||||
links.append(link)
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
|
@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
strip=strip,
|
||||
canonicalized=canonicalize
|
||||
)
|
||||
super(LxmlLinkExtractor, self).__init__(
|
||||
super().__init__(
|
||||
link_extractor=lx,
|
||||
allow=allow,
|
||||
deny=deny,
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor
|
||||
|
||||
|
||||
linkre = re.compile(
|
||||
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
|
||||
re.DOTALL | re.IGNORECASE)
|
||||
|
||||
|
||||
def clean_link(link_text):
|
||||
"""Remove leading and trailing whitespace and punctuation"""
|
||||
return link_text.strip("\t\r\n '\"\x0c")
|
||||
|
||||
|
||||
class RegexLinkExtractor(SgmlLinkExtractor):
|
||||
"""High performant link extractor"""
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
|
||||
def clean_text(text):
|
||||
return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
|
||||
|
||||
def clean_url(url):
|
||||
clean_url = ''
|
||||
try:
|
||||
clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
|
||||
except ValueError:
|
||||
pass
|
||||
return clean_url
|
||||
|
||||
if base_url is None:
|
||||
base_url = get_base_url(response_text, response_url, response_encoding)
|
||||
|
||||
links_text = linkre.findall(response_text)
|
||||
return [Link(clean_url(url).encode(response_encoding),
|
||||
clean_text(text))
|
||||
for url, _, text in links_text]
|
||||
|
|
@ -1,151 +0,0 @@
|
|||
"""
|
||||
SGMLParser-based Link extractors
|
||||
"""
|
||||
import warnings
|
||||
from urllib.parse import urljoin
|
||||
from sgmllib import SGMLParser
|
||||
|
||||
from w3lib.url import safe_url_string, canonicalize_url
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list, to_unicode
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
class BaseSgmlLinkExtractor(SGMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", unique=False, process_value=None,
|
||||
strip=True, canonicalized=False):
|
||||
warnings.warn(
|
||||
"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
|
||||
"Please use scrapy.linkextractors.LinkExtractor",
|
||||
ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
SGMLParser.__init__(self)
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_value = (lambda v: v) if process_value is None else process_value
|
||||
self.current_link = None
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
if canonicalized:
|
||||
self.link_key = lambda link: link.url
|
||||
else:
|
||||
self.link_key = lambda link: canonicalize_url(link.url,
|
||||
keep_fragments=True)
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
|
||||
""" Do the real extraction work """
|
||||
self.reset()
|
||||
self.feed(response_text)
|
||||
self.close()
|
||||
|
||||
ret = []
|
||||
if base_url is None:
|
||||
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in self.links:
|
||||
if isinstance(link.url, str):
|
||||
link.url = link.url.encode(response_encoding)
|
||||
try:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
except ValueError:
|
||||
continue
|
||||
link.url = safe_url_string(link.url, response_encoding)
|
||||
link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def _process_links(self, links):
|
||||
""" Normalize and filter extracted links
|
||||
|
||||
The subclass should override it if necessary
|
||||
"""
|
||||
return unique_list(links, key=self.link_key) if self.unique else links
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
links = self._extract_links(response.body, response.url, response.encoding)
|
||||
links = self._process_links(links)
|
||||
return links
|
||||
|
||||
def reset(self):
|
||||
SGMLParser.reset(self)
|
||||
self.links = []
|
||||
self.base_url = None
|
||||
self.current_link = None
|
||||
|
||||
def unknown_starttag(self, tag, attrs):
|
||||
if tag == 'base':
|
||||
self.base_url = dict(attrs).get('href')
|
||||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if self.strip and value is not None:
|
||||
value = strip_html5_whitespace(value)
|
||||
url = self.process_value(value)
|
||||
if url is not None:
|
||||
link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
|
||||
def unknown_endtag(self, tag):
|
||||
if self.scan_tag(tag):
|
||||
self.current_link = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.current_link:
|
||||
self.current_link.text = self.current_link.text + data
|
||||
|
||||
def matches(self, url):
|
||||
"""This extractor matches with any url, since
|
||||
it doesn't contain any patterns"""
|
||||
return True
|
||||
|
||||
|
||||
class SgmlLinkExtractor(FilteringLinkExtractor):
|
||||
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
|
||||
process_value=None, deny_extensions=None, restrict_css=(),
|
||||
strip=True, restrict_text=()):
|
||||
warnings.warn(
|
||||
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
|
||||
"Please use scrapy.linkextractors.LinkExtractor",
|
||||
ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
|
||||
unique=unique, process_value=process_value, strip=strip,
|
||||
canonicalized=canonicalize)
|
||||
|
||||
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
|
||||
allow_domains=allow_domains, deny_domains=deny_domains,
|
||||
restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
|
||||
canonicalize=canonicalize, deny_extensions=deny_extensions,
|
||||
restrict_text=restrict_text)
|
||||
|
||||
def extract_links(self, response):
|
||||
base_url = None
|
||||
if self.restrict_xpaths:
|
||||
base_url = get_base_url(response)
|
||||
body = u''.join(f
|
||||
for x in self.restrict_xpaths
|
||||
for f in response.xpath(x).getall()
|
||||
).encode(response.encoding, errors='xmlcharrefreplace')
|
||||
else:
|
||||
body = response.body
|
||||
|
||||
links = self._extract_links(body, response.url, response.encoding, base_url)
|
||||
links = self._process_links(links)
|
||||
return links
|
||||
|
|
@ -3,217 +3,86 @@ Item Loader
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
import itemloaders
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.loader.processors import Identity
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.utils.misc import arg_to_iter, extract_regex
|
||||
from scrapy.utils.python import flatten
|
||||
|
||||
|
||||
def unbound_method(method):
|
||||
class ItemLoader(itemloaders.ItemLoader):
|
||||
"""
|
||||
Allow to use single-argument functions as input or output processors
|
||||
(no need to define an unused first 'self' argument)
|
||||
A user-friendly abstraction to populate an :ref:`item <topics-items>` with data
|
||||
by applying :ref:`field processors <topics-loaders-processors>` to scraped data.
|
||||
When instantiated with a ``selector`` or a ``response`` it supports
|
||||
data extraction from web pages using :ref:`selectors <topics-selectors>`.
|
||||
|
||||
:param item: The item instance to populate using subsequent calls to
|
||||
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
|
||||
or :meth:`~ItemLoader.add_value`.
|
||||
:type item: scrapy.item.Item
|
||||
|
||||
:param selector: The selector to extract data from, when using the
|
||||
:meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
|
||||
:meth:`replace_css` method.
|
||||
:type selector: :class:`~scrapy.selector.Selector` object
|
||||
|
||||
:param response: The response used to construct the selector using the
|
||||
:attr:`default_selector_class`, unless the selector argument is given,
|
||||
in which case this argument is ignored.
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
If no item is given, one is instantiated automatically using the class in
|
||||
:attr:`default_item_class`.
|
||||
|
||||
The item, selector, response and remaining keyword arguments are
|
||||
assigned to the Loader context (accessible through the :attr:`context` attribute).
|
||||
|
||||
.. attribute:: item
|
||||
|
||||
The item object being parsed by this Item Loader.
|
||||
This is mostly used as a property so, when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
The currently active :ref:`Context <loaders-context>` of this Item Loader.
|
||||
|
||||
.. attribute:: default_item_class
|
||||
|
||||
An :ref:`item <topics-items>` class (or factory), used to instantiate
|
||||
items when not given in the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
The default input processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_output_processor
|
||||
|
||||
The default output processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
"""
|
||||
with suppress(AttributeError):
|
||||
if '.' not in method.__qualname__:
|
||||
return method.__func__
|
||||
return method
|
||||
|
||||
|
||||
class ItemLoader:
|
||||
|
||||
default_item_class = Item
|
||||
default_input_processor = Identity()
|
||||
default_output_processor = Identity()
|
||||
default_selector_class = Selector
|
||||
|
||||
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
|
||||
if selector is None and response is not None:
|
||||
selector = self.default_selector_class(response)
|
||||
self.selector = selector
|
||||
context.update(selector=selector, response=response)
|
||||
if item is None:
|
||||
item = self.default_item_class()
|
||||
self.context = context
|
||||
self.parent = parent
|
||||
self._local_item = context['item'] = item
|
||||
self._local_values = defaultdict(list)
|
||||
# values from initial item
|
||||
for field_name, value in ItemAdapter(item).items():
|
||||
self._values[field_name] += arg_to_iter(value)
|
||||
|
||||
@property
|
||||
def _values(self):
|
||||
if self.parent is not None:
|
||||
return self.parent._values
|
||||
else:
|
||||
return self._local_values
|
||||
|
||||
@property
|
||||
def item(self):
|
||||
if self.parent is not None:
|
||||
return self.parent.item
|
||||
else:
|
||||
return self._local_item
|
||||
|
||||
def nested_xpath(self, xpath, **context):
|
||||
selector = self.selector.xpath(xpath)
|
||||
context.update(selector=selector)
|
||||
subloader = self.__class__(
|
||||
item=self.item, parent=self, **context
|
||||
)
|
||||
return subloader
|
||||
|
||||
def nested_css(self, css, **context):
|
||||
selector = self.selector.css(css)
|
||||
context.update(selector=selector)
|
||||
subloader = self.__class__(
|
||||
item=self.item, parent=self, **context
|
||||
)
|
||||
return subloader
|
||||
|
||||
def add_value(self, field_name, value, *processors, **kw):
|
||||
value = self.get_value(value, *processors, **kw)
|
||||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in value.items():
|
||||
self._add_value(k, v)
|
||||
else:
|
||||
self._add_value(field_name, value)
|
||||
|
||||
def replace_value(self, field_name, value, *processors, **kw):
|
||||
value = self.get_value(value, *processors, **kw)
|
||||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in value.items():
|
||||
self._replace_value(k, v)
|
||||
else:
|
||||
self._replace_value(field_name, value)
|
||||
|
||||
def _add_value(self, field_name, value):
|
||||
value = arg_to_iter(value)
|
||||
processed_value = self._process_input_value(field_name, value)
|
||||
if processed_value:
|
||||
self._values[field_name] += arg_to_iter(processed_value)
|
||||
|
||||
def _replace_value(self, field_name, value):
|
||||
self._values.pop(field_name, None)
|
||||
self._add_value(field_name, value)
|
||||
|
||||
def get_value(self, value, *processors, **kw):
|
||||
regex = kw.get('re', None)
|
||||
if regex:
|
||||
value = arg_to_iter(value)
|
||||
value = flatten(extract_regex(regex, x) for x in value)
|
||||
|
||||
for proc in processors:
|
||||
if value is None:
|
||||
break
|
||||
_proc = proc
|
||||
proc = wrap_loader_context(proc, self.context)
|
||||
try:
|
||||
value = proc(value)
|
||||
except Exception as e:
|
||||
raise ValueError("Error with processor %s value=%r error='%s: %s'" %
|
||||
(_proc.__class__.__name__, value,
|
||||
type(e).__name__, str(e)))
|
||||
return value
|
||||
|
||||
def load_item(self):
|
||||
adapter = ItemAdapter(self.item)
|
||||
for field_name in tuple(self._values):
|
||||
value = self.get_output_value(field_name)
|
||||
if value is not None:
|
||||
adapter[field_name] = value
|
||||
return adapter.item
|
||||
|
||||
def get_output_value(self, field_name):
|
||||
proc = self.get_output_processor(field_name)
|
||||
proc = wrap_loader_context(proc, self.context)
|
||||
try:
|
||||
return proc(self._values[field_name])
|
||||
except Exception as e:
|
||||
raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
|
||||
(field_name, self._values[field_name], type(e).__name__, str(e)))
|
||||
|
||||
def get_collected_values(self, field_name):
|
||||
return self._values[field_name]
|
||||
|
||||
def get_input_processor(self, field_name):
|
||||
proc = getattr(self, '%s_in' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor',
|
||||
self.default_input_processor)
|
||||
return unbound_method(proc)
|
||||
|
||||
def get_output_processor(self, field_name):
|
||||
proc = getattr(self, '%s_out' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor',
|
||||
self.default_output_processor)
|
||||
return unbound_method(proc)
|
||||
|
||||
def _process_input_value(self, field_name, value):
|
||||
proc = self.get_input_processor(field_name)
|
||||
_proc = proc
|
||||
proc = wrap_loader_context(proc, self.context)
|
||||
try:
|
||||
return proc(value)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
"Error with input processor %s: field=%r value=%r "
|
||||
"error='%s: %s'" % (_proc.__class__.__name__, field_name,
|
||||
value, type(e).__name__, str(e)))
|
||||
|
||||
def _get_item_field_attr(self, field_name, key, default=None):
|
||||
field_meta = ItemAdapter(self.item).get_field_meta(field_name)
|
||||
return field_meta.get(key, default)
|
||||
|
||||
def _check_selector_method(self):
|
||||
if self.selector is None:
|
||||
raise RuntimeError("To use XPath or CSS selectors, "
|
||||
"%s must be instantiated with a selector "
|
||||
"or a response" % self.__class__.__name__)
|
||||
|
||||
def add_xpath(self, field_name, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
self.add_value(field_name, values, *processors, **kw)
|
||||
|
||||
def replace_xpath(self, field_name, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
self.replace_value(field_name, values, *processors, **kw)
|
||||
|
||||
def get_xpath(self, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
return self.get_value(values, *processors, **kw)
|
||||
|
||||
def _get_xpathvalues(self, xpaths, **kw):
|
||||
self._check_selector_method()
|
||||
xpaths = arg_to_iter(xpaths)
|
||||
return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
|
||||
|
||||
def add_css(self, field_name, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
self.add_value(field_name, values, *processors, **kw)
|
||||
|
||||
def replace_css(self, field_name, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
self.replace_value(field_name, values, *processors, **kw)
|
||||
|
||||
def get_css(self, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
return self.get_value(values, *processors, **kw)
|
||||
|
||||
def _get_cssvalues(self, csss, **kw):
|
||||
self._check_selector_method()
|
||||
csss = arg_to_iter(csss)
|
||||
return flatten(self.selector.css(css).getall() for css in csss)
|
||||
context.update(response=response)
|
||||
super().__init__(item=item, selector=selector, parent=parent, **context)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,21 @@
|
|||
"""Common functions used in Item Loaders code"""
|
||||
|
||||
from functools import partial
|
||||
from scrapy.utils.python import get_func_args
|
||||
import warnings
|
||||
|
||||
from itemloaders import common
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
def wrap_loader_context(function, context):
|
||||
"""Wrap functions that receive loader_context to contain the context
|
||||
"pre-loaded" and expose a interface that receives only one argument
|
||||
"""
|
||||
if 'loader_context' in get_func_args(function):
|
||||
return partial(function, loader_context=context)
|
||||
else:
|
||||
return function
|
||||
warnings.warn(
|
||||
"scrapy.loader.common.wrap_loader_context has moved to a new library."
|
||||
"Please update your reference to itemloaders.common.wrap_loader_context",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
return common.wrap_loader_context(function, context)
|
||||
|
|
|
|||
|
|
@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders.
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from collections import ChainMap
|
||||
from itemloaders import processors
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
|
||||
class MapCompose:
|
||||
MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)
|
||||
|
||||
def __init__(self, *functions, **default_loader_context):
|
||||
self.functions = functions
|
||||
self.default_loader_context = default_loader_context
|
||||
Compose = create_deprecated_class('Compose', processors.Compose)
|
||||
|
||||
def __call__(self, value, loader_context=None):
|
||||
values = arg_to_iter(value)
|
||||
if loader_context:
|
||||
context = ChainMap(loader_context, self.default_loader_context)
|
||||
else:
|
||||
context = self.default_loader_context
|
||||
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
|
||||
for func in wrapped_funcs:
|
||||
next_values = []
|
||||
for v in values:
|
||||
try:
|
||||
next_values += arg_to_iter(func(v))
|
||||
except Exception as e:
|
||||
raise ValueError("Error in MapCompose with "
|
||||
"%s value=%r error='%s: %s'" %
|
||||
(str(func), value, type(e).__name__,
|
||||
str(e)))
|
||||
values = next_values
|
||||
return values
|
||||
TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)
|
||||
|
||||
Identity = create_deprecated_class('Identity', processors.Identity)
|
||||
|
||||
class Compose:
|
||||
SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)
|
||||
|
||||
def __init__(self, *functions, **default_loader_context):
|
||||
self.functions = functions
|
||||
self.stop_on_none = default_loader_context.get('stop_on_none', True)
|
||||
self.default_loader_context = default_loader_context
|
||||
|
||||
def __call__(self, value, loader_context=None):
|
||||
if loader_context:
|
||||
context = ChainMap(loader_context, self.default_loader_context)
|
||||
else:
|
||||
context = self.default_loader_context
|
||||
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
|
||||
for func in wrapped_funcs:
|
||||
if value is None and self.stop_on_none:
|
||||
break
|
||||
try:
|
||||
value = func(value)
|
||||
except Exception as e:
|
||||
raise ValueError("Error in Compose with "
|
||||
"%s value=%r error='%s: %s'" %
|
||||
(str(func), value, type(e).__name__, str(e)))
|
||||
return value
|
||||
|
||||
|
||||
class TakeFirst:
|
||||
|
||||
def __call__(self, values):
|
||||
for value in values:
|
||||
if value is not None and value != '':
|
||||
return value
|
||||
|
||||
|
||||
class Identity:
|
||||
|
||||
def __call__(self, values):
|
||||
return values
|
||||
|
||||
|
||||
class SelectJmes:
|
||||
"""
|
||||
Query the input string for the jmespath (given at instantiation),
|
||||
and return the answer
|
||||
Requires : jmespath(https://github.com/jmespath/jmespath)
|
||||
Note: SelectJmes accepts only one input element at a time.
|
||||
"""
|
||||
def __init__(self, json_path):
|
||||
self.json_path = json_path
|
||||
import jmespath
|
||||
self.compiled_path = jmespath.compile(self.json_path)
|
||||
|
||||
def __call__(self, value):
|
||||
"""Query value for the jmespath query and return answer
|
||||
:param value: a data structure (dict, list) to extract from
|
||||
:return: Element extracted according to jmespath query
|
||||
"""
|
||||
return self.compiled_path.search(value)
|
||||
|
||||
|
||||
class Join:
|
||||
|
||||
def __init__(self, separator=u' '):
|
||||
self.separator = separator
|
||||
|
||||
def __call__(self, values):
|
||||
return self.separator.join(values)
|
||||
Join = create_deprecated_class('Join', processors.Join)
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class LogFormatter:
|
|||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
'level': logging.INFO, # lowering the level from logging.WARNING
|
||||
'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'args': {
|
||||
'exception': exception,
|
||||
'item': item,
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
|
|||
resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
|
||||
)
|
||||
|
||||
super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
|
||||
super().__init__(download_func=download_func, settings=settings)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
|
|
|
|||
|
|
@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
DEFAULT_IMAGES_RESULT_FIELD = 'images'
|
||||
|
||||
def __init__(self, store_uri, download_func=None, settings=None):
|
||||
super(ImagesPipeline, self).__init__(store_uri, settings=settings,
|
||||
download_func=download_func)
|
||||
super().__init__(store_uri, settings=settings, download_func=download_func)
|
||||
|
||||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
|
|||
"""
|
||||
|
||||
def __init__(self, reactor, cache_size, timeout):
|
||||
super(CachingThreadedResolver, self).__init__(reactor)
|
||||
super().__init__(reactor)
|
||||
dnscache.limit = cache_size
|
||||
self.timeout = timeout
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
|
|||
# so the input argument above is simply overridden
|
||||
# to enforce Scrapy's DNS_TIMEOUT setting's value
|
||||
timeout = (self.timeout,)
|
||||
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
|
||||
d = super().getHostByName(name, timeout)
|
||||
if dnscache.limit:
|
||||
d.addCallback(self._cache_result, name)
|
||||
return d
|
||||
|
|
@ -80,16 +80,16 @@ class CachingHostnameResolver:
|
|||
class CachingResolutionReceiver(resolutionReceiver):
|
||||
|
||||
def resolutionBegan(self, resolution):
|
||||
super(CachingResolutionReceiver, self).resolutionBegan(resolution)
|
||||
super().resolutionBegan(resolution)
|
||||
self.resolution = resolution
|
||||
self.resolved = False
|
||||
|
||||
def addressResolved(self, address):
|
||||
super(CachingResolutionReceiver, self).addressResolved(address)
|
||||
super().addressResolved(address)
|
||||
self.resolved = True
|
||||
|
||||
def resolutionComplete(self):
|
||||
super(CachingResolutionReceiver, self).resolutionComplete()
|
||||
super().resolutionComplete()
|
||||
if self.resolved:
|
||||
dnscache[hostName] = self.resolution
|
||||
|
||||
|
|
|
|||
|
|
@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
|
|||
kwargs.setdefault('base_url', response.url)
|
||||
|
||||
self.response = response
|
||||
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
|
||||
super().__init__(text=text, type=st, root=root, **kwargs)
|
||||
|
|
|
|||
|
|
@ -53,8 +53,7 @@ class SettingsAttribute:
|
|||
self.priority = priority
|
||||
|
||||
def __str__(self):
|
||||
return "<SettingsAttribute value={self.value!r} " \
|
||||
"priority={self.priority}>".format(self=self)
|
||||
return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
|
|
@ -84,7 +83,8 @@ class BaseSettings(MutableMapping):
|
|||
def __init__(self, values=None, priority='project'):
|
||||
self.frozen = False
|
||||
self.attributes = {}
|
||||
self.update(values, priority)
|
||||
if values:
|
||||
self.update(values, priority)
|
||||
|
||||
def __getitem__(self, opt_name):
|
||||
if opt_name not in self:
|
||||
|
|
@ -473,7 +473,7 @@ class Settings(BaseSettings):
|
|||
# Do not pass kwarg values here. We don't want to promote user-defined
|
||||
# dicts, and we want to update, not replace, default dicts with the
|
||||
# values given by the user
|
||||
super(Settings, self).__init__()
|
||||
super().__init__()
|
||||
self.setmodule(default_settings, 'default')
|
||||
# Promote default dictionaries to BaseSettings instances for per-key
|
||||
# priorities
|
||||
|
|
|
|||
|
|
@ -142,10 +142,12 @@ FEED_STORAGES = {}
|
|||
FEED_STORAGES_BASE = {
|
||||
'': 'scrapy.extensions.feedexport.FileFeedStorage',
|
||||
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
|
||||
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
|
||||
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
|
||||
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
|
||||
'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
|
||||
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
|
||||
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
|
||||
}
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT = 0
|
||||
FEED_EXPORTERS = {}
|
||||
FEED_EXPORTERS_BASE = {
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
|
|
@ -159,6 +161,7 @@ FEED_EXPORTERS_BASE = {
|
|||
FEED_EXPORT_INDENT = 0
|
||||
|
||||
FEED_STORAGE_FTP_ACTIVE = False
|
||||
FEED_STORAGE_GCS_ACL = ''
|
||||
FEED_STORAGE_S3_ACL = ''
|
||||
|
||||
FILES_STORE_S3_ACL = 'private'
|
||||
|
|
@ -168,6 +171,8 @@ FTP_USER = 'anonymous'
|
|||
FTP_PASSWORD = 'guest'
|
||||
FTP_PASSIVE_MODE = True
|
||||
|
||||
GCS_PROJECT_ID = None
|
||||
|
||||
HTTPCACHE_ENABLED = False
|
||||
HTTPCACHE_DIR = 'httpcache'
|
||||
HTTPCACHE_IGNORE_MISSING = False
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
|
|||
|
||||
def __init__(self, response, *args, **kwargs):
|
||||
self.response = response
|
||||
super(HttpError, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class HttpErrorMiddleware:
|
||||
|
|
|
|||
|
|
@ -86,7 +86,10 @@ class Spider(object_ref):
|
|||
)
|
||||
return Request(url, dont_filter=True)
|
||||
|
||||
def parse(self, response):
|
||||
def _parse(self, response, **kwargs):
|
||||
return self.parse(response, **kwargs)
|
||||
|
||||
def parse(self, response, **kwargs):
|
||||
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -75,13 +75,18 @@ class CrawlSpider(Spider):
|
|||
rules = ()
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(CrawlSpider, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
self._compile_rules()
|
||||
|
||||
def parse(self, response):
|
||||
return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
|
||||
def _parse(self, response, **kwargs):
|
||||
return self._parse_response(
|
||||
response=response,
|
||||
callback=self.parse_start_url,
|
||||
cb_kwargs=kwargs,
|
||||
follow=True,
|
||||
)
|
||||
|
||||
def parse_start_url(self, response):
|
||||
def parse_start_url(self, response, **kwargs):
|
||||
return []
|
||||
|
||||
def process_results(self, response, results):
|
||||
|
|
@ -140,6 +145,6 @@ class CrawlSpider(Spider):
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
|
||||
return spider
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
|
|||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
||||
def parse(self, response):
|
||||
def _parse(self, response, **kwargs):
|
||||
if not hasattr(self, 'parse_node'):
|
||||
raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
|
||||
|
||||
|
|
@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
|
|||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
||||
def parse(self, response):
|
||||
def _parse(self, response, **kwargs):
|
||||
if not hasattr(self, 'parse_row'):
|
||||
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
|
||||
response = self.adapt_response(response)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ class InitSpider(Spider):
|
|||
"""Base Spider with initialization facilities"""
|
||||
|
||||
def start_requests(self):
|
||||
self._postinit_reqs = super(InitSpider, self).start_requests()
|
||||
self._postinit_reqs = super().start_requests()
|
||||
return iterate_spider_output(self.init_request())
|
||||
|
||||
def initialized(self, response=None):
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class SitemapSpider(Spider):
|
|||
sitemap_alternate_links = False
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(SitemapSpider, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
self._cbs = []
|
||||
for r, c in self.sitemap_rules:
|
||||
if isinstance(c, str):
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
|
|||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
|
||||
super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
|
||||
super().__init__(path, *args, **kwargs)
|
||||
|
||||
return DirectoriesCreated
|
||||
|
||||
|
|
@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
|
|||
|
||||
def push(self, obj):
|
||||
s = serialize(obj)
|
||||
super(SerializableQueue, self).push(s)
|
||||
super().push(s)
|
||||
|
||||
def pop(self):
|
||||
s = super(SerializableQueue, self).pop()
|
||||
s = super().pop()
|
||||
if s:
|
||||
return deserialize(s)
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
|
|||
|
||||
def __init__(self, crawler, key):
|
||||
self.spider = crawler.spider
|
||||
super(ScrapyRequestQueue, self).__init__(key)
|
||||
super().__init__(key)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, key, *args, **kwargs):
|
||||
|
|
@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
|
|||
|
||||
def push(self, request):
|
||||
request = request_to_dict(request, self.spider)
|
||||
return super(ScrapyRequestQueue, self).push(request)
|
||||
return super().push(request)
|
||||
|
||||
def pop(self):
|
||||
request = super(ScrapyRequestQueue, self).pop()
|
||||
request = super().pop()
|
||||
|
||||
if not request:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class StatsCollector:
|
|||
class MemoryStatsCollector(StatsCollector):
|
||||
|
||||
def __init__(self, crawler):
|
||||
super(MemoryStatsCollector, self).__init__(crawler)
|
||||
super().__init__(crawler)
|
||||
self.spider_stats = {}
|
||||
|
||||
def _persist_stats(self, stats, spider):
|
||||
|
|
|
|||
|
|
@ -28,8 +28,7 @@ class Root(Resource):
|
|||
|
||||
|
||||
def _getarg(request, name, default=None, type=str):
|
||||
return type(request.args[name][0]) \
|
||||
if name in request.args else default
|
||||
return type(request.args[name][0]) if name in request.args else default
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -101,11 +101,13 @@ def get_config(use_closest=True):
|
|||
|
||||
|
||||
def get_sources(use_closest=True):
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
|
||||
os.path.expanduser('~/.config')
|
||||
sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
|
||||
xdg_config_home + '/scrapy.cfg',
|
||||
os.path.expanduser('~/.scrapy.cfg')]
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
|
||||
sources = [
|
||||
'/etc/scrapy.cfg',
|
||||
r'c:\scrapy\scrapy.cfg',
|
||||
xdg_config_home + '/scrapy.cfg',
|
||||
os.path.expanduser('~/.scrapy.cfg'),
|
||||
]
|
||||
if use_closest:
|
||||
sources.append(closest_scrapy_cfg())
|
||||
return sources
|
||||
|
|
@ -113,6 +115,7 @@ def get_sources(use_closest=True):
|
|||
|
||||
def feed_complete_default_values_from_settings(feed, settings):
|
||||
out = feed.copy()
|
||||
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
|
||||
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
|
||||
out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
|
||||
|
|
|
|||
|
|
@ -9,16 +9,15 @@ from w3lib.http import basic_auth_header
|
|||
|
||||
class CurlParser(argparse.ArgumentParser):
|
||||
def error(self, message):
|
||||
error_msg = \
|
||||
'There was an error parsing the curl command: {}'.format(message)
|
||||
error_msg = 'There was an error parsing the curl command: {}'.format(message)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
|
||||
curl_parser = CurlParser()
|
||||
curl_parser.add_argument('url')
|
||||
curl_parser.add_argument('-H', '--header', dest='headers', action='append')
|
||||
curl_parser.add_argument('-X', '--request', dest='method', default='get')
|
||||
curl_parser.add_argument('-d', '--data', dest='data')
|
||||
curl_parser.add_argument('-X', '--request', dest='method')
|
||||
curl_parser.add_argument('-d', '--data', '--data-raw', dest='data')
|
||||
curl_parser.add_argument('-u', '--user', dest='auth')
|
||||
|
||||
|
||||
|
|
@ -40,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
|
|||
|
||||
:param str curl_command: string containing the curl command
|
||||
:param bool ignore_unknown_options: If true, only a warning is emitted when
|
||||
cURL options are unknown. Otherwise raises an error. (default: True)
|
||||
cURL options are unknown. Otherwise
|
||||
raises an error. (default: True)
|
||||
:return: dictionary of Request kwargs
|
||||
"""
|
||||
|
||||
|
|
@ -66,7 +66,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
|
|||
if not parsed_url.scheme:
|
||||
url = 'http://' + url
|
||||
|
||||
result = {'method': parsed_args.method.upper(), 'url': url}
|
||||
method = parsed_args.method or 'GET'
|
||||
|
||||
result = {'method': method.upper(), 'url': url}
|
||||
|
||||
headers = []
|
||||
cookies = {}
|
||||
|
|
@ -90,5 +92,9 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
|
|||
result['cookies'] = cookies
|
||||
if parsed_args.data:
|
||||
result['body'] = parsed_args.data
|
||||
if not parsed_args.method:
|
||||
# if the "data" is specified but the "method" is not specified,
|
||||
# the default method is 'POST'
|
||||
result['method'] = 'POST'
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class CaselessDict(dict):
|
|||
__slots__ = ()
|
||||
|
||||
def __init__(self, seq=None):
|
||||
super(CaselessDict, self).__init__()
|
||||
super().__init__()
|
||||
if seq:
|
||||
self.update(seq)
|
||||
|
||||
|
|
@ -53,7 +53,7 @@ class CaselessDict(dict):
|
|||
def update(self, seq):
|
||||
seq = seq.items() if isinstance(seq, Mapping) else seq
|
||||
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
|
||||
super(CaselessDict, self).update(iseq)
|
||||
super().update(iseq)
|
||||
|
||||
@classmethod
|
||||
def fromkeys(cls, keys, value=None):
|
||||
|
|
@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
|
|||
"""
|
||||
|
||||
def __init__(self, limit=None):
|
||||
super(LocalCache, self).__init__()
|
||||
super().__init__()
|
||||
self.limit = limit
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if self.limit:
|
||||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
super(LocalCache, self).__setitem__(key, value)
|
||||
super().__setitem__(key, value)
|
||||
|
||||
|
||||
class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
|
||||
|
|
@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
|
|||
"""
|
||||
|
||||
def __init__(self, limit=None):
|
||||
super(LocalWeakReferencedCache, self).__init__()
|
||||
super().__init__()
|
||||
self.data = LocalCache(limit=limit)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
try:
|
||||
super(LocalWeakReferencedCache, self).__setitem__(key, value)
|
||||
super().__setitem__(key, value)
|
||||
except TypeError:
|
||||
pass # key is not weak-referenceable, skip caching
|
||||
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return super(LocalWeakReferencedCache, self).__getitem__(key)
|
||||
return super().__getitem__(key)
|
||||
except (TypeError, KeyError):
|
||||
return None # key is either not weak-referenceable or not cached
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ def create_deprecated_class(
|
|||
warned_on_subclass = False
|
||||
|
||||
def __new__(metacls, name, bases, clsdict_):
|
||||
cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
|
||||
cls = super().__new__(metacls, name, bases, clsdict_)
|
||||
if metacls.deprecated_class is None:
|
||||
metacls.deprecated_class = cls
|
||||
return cls
|
||||
|
|
@ -73,7 +73,7 @@ def create_deprecated_class(
|
|||
if warn_once:
|
||||
msg += ' (warning only on first subclass, there may be others)'
|
||||
warnings.warn(msg, warn_category, stacklevel=2)
|
||||
super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
|
||||
super().__init__(name, bases, clsdict_)
|
||||
|
||||
# see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
|
||||
# and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
|
||||
|
|
@ -88,7 +88,7 @@ def create_deprecated_class(
|
|||
# is the deprecated class itself - subclasses of the
|
||||
# deprecated class should not use custom `__subclasscheck__`
|
||||
# method.
|
||||
return super(DeprecatedClass, cls).__subclasscheck__(sub)
|
||||
return super().__subclasscheck__(sub)
|
||||
|
||||
if not inspect.isclass(sub):
|
||||
raise TypeError("issubclass() arg 1 must be a class")
|
||||
|
|
@ -102,7 +102,7 @@ def create_deprecated_class(
|
|||
msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
|
||||
new=_clspath(new_class, new_class_path))
|
||||
warnings.warn(msg, warn_category, stacklevel=2)
|
||||
return super(DeprecatedClass, cls).__call__(*args, **kwargs)
|
||||
return super().__call__(*args, **kwargs)
|
||||
|
||||
deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
|
||||
|
||||
|
|
|
|||
|
|
@ -2,20 +2,42 @@
|
|||
pprint and pformat wrappers with colorization support
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import platform
|
||||
import sys
|
||||
from distutils.version import LooseVersion as parse_version
|
||||
from pprint import pformat as pformat_
|
||||
|
||||
|
||||
def _enable_windows_terminal_processing():
|
||||
# https://stackoverflow.com/a/36760881
|
||||
kernel32 = ctypes.windll.kernel32
|
||||
return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
|
||||
|
||||
|
||||
def _tty_supports_color():
|
||||
if sys.platform != "win32":
|
||||
return True
|
||||
|
||||
if parse_version(platform.version()) < parse_version("10.0.14393"):
|
||||
return True
|
||||
|
||||
# Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
|
||||
# processing is enabled.
|
||||
return _enable_windows_terminal_processing()
|
||||
|
||||
|
||||
def _colorize(text, colorize=True):
|
||||
if not colorize or not sys.stdout.isatty():
|
||||
if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
|
||||
return text
|
||||
try:
|
||||
from pygments import highlight
|
||||
except ImportError:
|
||||
return text
|
||||
else:
|
||||
from pygments.formatters import TerminalFormatter
|
||||
from pygments.lexers import PythonLexer
|
||||
return highlight(text, PythonLexer(), TerminalFormatter())
|
||||
except ImportError:
|
||||
return text
|
||||
|
||||
|
||||
def pformat(obj, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -176,7 +176,7 @@ class LogCounterHandler(logging.Handler):
|
|||
"""Record log levels count into a crawler stats"""
|
||||
|
||||
def __init__(self, crawler, *args, **kwargs):
|
||||
super(LogCounterHandler, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.crawler = crawler
|
||||
|
||||
def emit(self, record):
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from w3lib.html import replace_entities
|
|||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
|
||||
|
|
@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'):
|
|||
* if the regex contains multiple numbered groups, all those will be returned (flattened)
|
||||
* if the regex doesn't contain any group the entire regex matching is returned
|
||||
"""
|
||||
warnings.warn(
|
||||
"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
if isinstance(regex, str):
|
||||
regex = re.compile(regex, re.UNICODE)
|
||||
|
|
@ -138,8 +144,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
|
|||
|
||||
Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
|
||||
|
||||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
||||
extension has not been implemented correctly).
|
||||
.. versionchanged:: 2.2
|
||||
Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
|
||||
extension has not been implemented correctly).
|
||||
"""
|
||||
if settings is None:
|
||||
if crawler is None:
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True):
|
|||
from twisted.internet import reactor
|
||||
reactor._handleSignals()
|
||||
signal.signal(signal.SIGTERM, function)
|
||||
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
|
||||
override_sigint:
|
||||
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
|
||||
signal.signal(signal.SIGINT, function)
|
||||
# Catch Ctrl-Break in windows
|
||||
if hasattr(signal, 'SIGBREAK'):
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@ import gc
|
|||
import inspect
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
import weakref
|
||||
from functools import partial, wraps
|
||||
from itertools import chain
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.decorators import deprecated
|
||||
|
||||
|
||||
|
|
@ -127,6 +129,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
|
|||
In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
|
||||
the start position of the match, and the ending (regarding the entire text).
|
||||
"""
|
||||
|
||||
def _chunk_iter():
|
||||
offset = len(text)
|
||||
while True:
|
||||
|
|
@ -158,6 +161,7 @@ def memoizemethod_noargs(method):
|
|||
if self not in cache:
|
||||
cache[self] = method(self, *args, **kwargs)
|
||||
return cache[self]
|
||||
|
||||
return new_method
|
||||
|
||||
|
||||
|
|
@ -276,6 +280,7 @@ def equal_attributes(obj1, obj2, attributes):
|
|||
class WeakKeyCache:
|
||||
|
||||
def __init__(self, default_factory):
|
||||
warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self.default_factory = default_factory
|
||||
self._weakdict = weakref.WeakKeyDictionary()
|
||||
|
||||
|
|
@ -285,6 +290,7 @@ class WeakKeyCache:
|
|||
return self._weakdict[key]
|
||||
|
||||
|
||||
@deprecated
|
||||
def retry_on_eintr(function, *args, **kw):
|
||||
"""Run a function and retry it while getting EINTR errors"""
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -47,13 +47,17 @@ def response_httprepr(response):
|
|||
is provided only for reference, since it's not the exact stream of bytes
|
||||
that was received (that's not exposed by Twisted).
|
||||
"""
|
||||
s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
|
||||
to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n"
|
||||
values = [
|
||||
b"HTTP/1.1 ",
|
||||
to_bytes(str(response.status)),
|
||||
b" ",
|
||||
to_bytes(http.RESPONSES.get(response.status, b'')),
|
||||
b"\r\n",
|
||||
]
|
||||
if response.headers:
|
||||
s += response.headers.to_string() + b"\r\n"
|
||||
s += b"\r\n"
|
||||
s += response.body
|
||||
return s
|
||||
values.extend([response.headers.to_string(), b"\r\n"])
|
||||
values.extend([b"\r\n", response.body])
|
||||
return b"".join(values)
|
||||
|
||||
|
||||
def open_in_browser(response, _openfunc=webbrowser.open):
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
|
|||
elif isinstance(o, Response):
|
||||
return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
|
||||
else:
|
||||
return super(ScrapyJSONEncoder, self).default(o)
|
||||
return super().default(o)
|
||||
|
||||
|
||||
class ScrapyJSONDecoder(json.JSONDecoder):
|
||||
|
|
|
|||
|
|
@ -34,10 +34,12 @@ def iter_spider_classes(module):
|
|||
from scrapy.spiders import Spider
|
||||
|
||||
for obj in vars(module).values():
|
||||
if inspect.isclass(obj) and \
|
||||
issubclass(obj, Spider) and \
|
||||
obj.__module__ == module.__name__ and \
|
||||
getattr(obj, 'name', None):
|
||||
if (
|
||||
inspect.isclass(obj)
|
||||
and issubclass(obj, Spider)
|
||||
and obj.__module__ == module.__name__
|
||||
and getattr(obj, 'name', None)
|
||||
):
|
||||
yield obj
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
This module contains some assorted functions used in tests
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from posixpath import split
|
||||
import asyncio
|
||||
import os
|
||||
from posixpath import split
|
||||
from unittest import mock
|
||||
|
||||
from importlib import import_module
|
||||
from twisted.trial.unittest import SkipTest
|
||||
|
|
@ -126,3 +126,19 @@ def get_from_asyncio_queue(value):
|
|||
getter = q.get()
|
||||
q.put_nowait(value)
|
||||
return getter
|
||||
|
||||
|
||||
def mock_google_cloud_storage():
|
||||
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
|
||||
classes and set their proper return values.
|
||||
"""
|
||||
from google.cloud.storage import Client, Bucket, Blob
|
||||
client_mock = mock.create_autospec(Client)
|
||||
|
||||
bucket_mock = mock.create_autospec(Bucket)
|
||||
client_mock.get_bucket.return_value = bucket_mock
|
||||
|
||||
blob_mock = mock.create_autospec(Blob)
|
||||
bucket_mock.blob.return_value = blob_mock
|
||||
|
||||
return (client_mock, bucket_mock, blob_mock)
|
||||
|
|
|
|||
|
|
@ -7,12 +7,12 @@ class SiteTest:
|
|||
|
||||
def setUp(self):
|
||||
from twisted.internet import reactor
|
||||
super(SiteTest, self).setUp()
|
||||
super().setUp()
|
||||
self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
|
||||
self.baseurl = "http://localhost:%d/" % self.site.getHost().port
|
||||
|
||||
def tearDown(self):
|
||||
super(SiteTest, self).tearDown()
|
||||
super().tearDown()
|
||||
self.site.stopListening()
|
||||
|
||||
def url(self, path):
|
||||
|
|
|
|||
|
|
@ -83,25 +83,54 @@ def add_http_if_no_scheme(url):
|
|||
return url
|
||||
|
||||
|
||||
def _is_posix_path(string):
|
||||
return bool(
|
||||
re.match(
|
||||
r'''
|
||||
^ # start with...
|
||||
(
|
||||
\. # ...a single dot,
|
||||
(
|
||||
\. | [^/\.]+ # optionally followed by
|
||||
)? # either a second dot or some characters
|
||||
|
|
||||
~ # $HOME
|
||||
)? # optional match of ".", ".." or ".blabla"
|
||||
/ # at least one "/" for a file path,
|
||||
. # and something after the "/"
|
||||
''',
|
||||
string,
|
||||
flags=re.VERBOSE,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_windows_path(string):
|
||||
return bool(
|
||||
re.match(
|
||||
r'''
|
||||
^
|
||||
(
|
||||
[a-z]:\\
|
||||
| \\\\
|
||||
)
|
||||
''',
|
||||
string,
|
||||
flags=re.IGNORECASE | re.VERBOSE,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _is_filesystem_path(string):
|
||||
return _is_posix_path(string) or _is_windows_path(string)
|
||||
|
||||
|
||||
def guess_scheme(url):
|
||||
"""Add an URL scheme if missing: file:// for filepath-like input or http:// otherwise."""
|
||||
parts = urlparse(url)
|
||||
if parts.scheme:
|
||||
return url
|
||||
# Note: this does not match Windows filepath
|
||||
if re.match(r'''^ # start with...
|
||||
(
|
||||
\. # ...a single dot,
|
||||
(
|
||||
\. | [^/\.]+ # optionally followed by
|
||||
)? # either a second dot or some characters
|
||||
)? # optional match of ".", ".." or ".blabla"
|
||||
/ # at least one "/" for a file path,
|
||||
. # and something after the "/"
|
||||
''', parts.path, flags=re.VERBOSE):
|
||||
"""Add an URL scheme if missing: file:// for filepath-like input or
|
||||
http:// otherwise."""
|
||||
if _is_filesystem_path(url):
|
||||
return any_to_uri(url)
|
||||
else:
|
||||
return add_http_if_no_scheme(url)
|
||||
return add_http_if_no_scheme(url)
|
||||
|
||||
|
||||
def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
|
||||
|
|
|
|||
171
setup.cfg
171
setup.cfg
|
|
@ -3,3 +3,174 @@ doc_files = docs AUTHORS INSTALL LICENSE README.rst
|
|||
|
||||
[bdist_wheel]
|
||||
universal=1
|
||||
|
||||
[mypy]
|
||||
ignore_missing_imports = true
|
||||
follow_imports = skip
|
||||
|
||||
# FIXME: remove the following sections once the issues are solved
|
||||
|
||||
[mypy-scrapy]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.commands]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.commands.bench]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.commands.parse]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.downloadermiddlewares.httpproxy]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.contracts]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.core.spidermw]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.interfaces]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.item]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.http.cookies]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.mail]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.pipelines.images]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.settings.default_settings]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.spidermiddlewares.referer]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.httpobj]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.request]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.response]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.spider]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.trackref]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.mocks.dummydbm]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.spiders]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.exception]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_cmdline_crawl_with_pipeline.test_spider.spiders.normal]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_command_fetch]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_command_parse]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_command_shell]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_command_version]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_contracts]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_crawler]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_downloader_handlers]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_engine]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_exporters]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_http_request]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_linkextractors]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_loader]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_loader_deprecated]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_pipeline_crawl]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_pipeline_files]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_pipeline_images]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_pipelines]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_cb_kwargs]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_left]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_scheduler]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_signals]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.nested.spider4]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.spider1]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.spider2]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spiderloader.test_spiders.spider3]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spidermiddleware_httperror]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spidermiddleware_output_chain]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_spidermiddleware_referer]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_reqser]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_serialize]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_spider]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_utils_url]
|
||||
ignore_errors = True
|
||||
|
|
|
|||
42
setup.py
42
setup.py
|
|
@ -18,12 +18,38 @@ def has_environment_marker_platform_impl_support():
|
|||
return parse_version(setuptools_version) >= parse_version('18.5')
|
||||
|
||||
|
||||
install_requires = [
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
'cssselect>=0.9.1',
|
||||
'itemloaders>=1.0.1',
|
||||
'parsel>=1.5.0',
|
||||
'PyDispatcher>=2.0.5',
|
||||
'pyOpenSSL>=16.2.0',
|
||||
'queuelib>=1.4.2',
|
||||
'service_identity>=16.0.0',
|
||||
'w3lib>=1.17.0',
|
||||
'zope.interface>=4.1.3',
|
||||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
]
|
||||
extras_require = {}
|
||||
|
||||
if has_environment_marker_platform_impl_support():
|
||||
extras_require[':platform_python_implementation == "CPython"'] = [
|
||||
'lxml>=3.5.0',
|
||||
]
|
||||
extras_require[':platform_python_implementation == "PyPy"'] = [
|
||||
# Earlier lxml versions are affected by
|
||||
# https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,
|
||||
# which was fixed in Cython 0.26, released on 2017-06-19, and used to
|
||||
# generate the C headers of lxml release tarballs published since then, the
|
||||
# first of which was:
|
||||
'lxml>=4.0.0',
|
||||
'PyPyDispatcher>=2.1.0',
|
||||
]
|
||||
else:
|
||||
install_requires.append('lxml>=3.5.0')
|
||||
|
||||
|
||||
setup(
|
||||
|
|
@ -67,20 +93,6 @@ setup(
|
|||
'Topic :: Software Development :: Libraries :: Python Modules',
|
||||
],
|
||||
python_requires='>=3.5.2',
|
||||
install_requires=[
|
||||
'Twisted>=17.9.0',
|
||||
'cryptography>=2.0',
|
||||
'cssselect>=0.9.1',
|
||||
'lxml>=3.5.0',
|
||||
'parsel>=1.5.0',
|
||||
'PyDispatcher>=2.0.5',
|
||||
'pyOpenSSL>=16.2.0',
|
||||
'queuelib>=1.4.2',
|
||||
'service_identity>=16.0.0',
|
||||
'w3lib>=1.17.0',
|
||||
'zope.interface>=4.1.3',
|
||||
'protego>=0.1.15',
|
||||
'itemadapter>=0.1.0',
|
||||
],
|
||||
install_requires=install_requires,
|
||||
extras_require=extras_require,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
from urllib.parse import urlparse
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.names.client import createResolver
|
||||
from twisted.names import cache, hosts as hostsModule, resolve
|
||||
from twisted.names.client import Resolver
|
||||
from twisted.python.runtime import platform
|
||||
|
||||
from scrapy import Spider, Request
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
|
|
@ -10,6 +12,16 @@ from scrapy.utils.log import configure_logging
|
|||
from tests.mockserver import MockServer, MockDNSServer
|
||||
|
||||
|
||||
# https://stackoverflow.com/a/32784190
|
||||
def createResolver(servers=None, resolvconf=None, hosts=None):
|
||||
if hosts is None:
|
||||
hosts = b'/etc/hosts' if platform.getType() == 'posix' else r'c:\windows\hosts'
|
||||
theResolver = Resolver(resolvconf, servers)
|
||||
hostResolver = hostsModule.Resolver(hosts)
|
||||
chain = [hostResolver, cache.CacheResolver(), theResolver]
|
||||
return resolve.ResolverChain(chain)
|
||||
|
||||
|
||||
class LocalhostSpider(Spider):
|
||||
name = "localhost_spider"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,3 @@
|
|||
scrapy/linkextractors/sgml.py
|
||||
scrapy/linkextractors/regex.py
|
||||
scrapy/linkextractors/htmlparser.py
|
||||
scrapy/downloadermiddlewares/cookies.py
|
||||
scrapy/extensions/statsmailer.py
|
||||
scrapy/extensions/memusage.py
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
-----BEGIN CERTIFICATE-----
|
||||
MIIDNzCCAh+gAwIBAgIJANWqWyPdTY8CMA0GCSqGSIb3DQEBCwUAMDIxCzAJBgNV
|
||||
BAYTAklFMQ8wDQYDVQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0x
|
||||
NzA0MjcxNzQxNTdaFw0xODA0MjcxNzQxNTdaMDIxCzAJBgNVBAYTAklFMQ8wDQYD
|
||||
VQQKDAZTY3JhcHkxEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEB
|
||||
BQADggEPADCCAQoCggEBAK1jcwlJ+bpr63lmK1mSk83nduF+27EPTU3RyteoPM2K
|
||||
o/RqZnr/mR29U6Pu42YuhLvBUu7rQxGi+rgkwno6lMFP4y5glxRygIlPsP4WQO3Y
|
||||
njmysWfYxQoIml2A+tiLewrMZocHI2cNgrO8Fd0u7KMiLlvUCN0pVyOwZ/ym9rPY
|
||||
ObfquG/xYTFzgYD/wy1n4AXE4ve3uZPfB3ZGtB3fUmuowg5KZ1L3uWpviyqr1qB/
|
||||
8NXcORLegAPsquLA05gnDPOuMs7dSMeKMphvpbSerRXLGxLIfWOZ0rs8oV96Re52
|
||||
gSEg/kIIS+ts37sJofcEnx9C4FkTR8zXin9eZhgCYs0CAwEAAaNQME4wHQYDVR0O
|
||||
BBYEFOoYbg0MvcnbTN0jxISsP2ctMbjpMB8GA1UdIwQYMBaAFOoYbg0MvcnbTN0j
|
||||
xISsP2ctMbjpMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAF/JlzES
|
||||
9Z3Azaj60gvJHyPJsPSM4tUfnWoFfFrui3oPG5TJPxWqrLBsTEachUTKOd5+XR2i
|
||||
jxUuREMkcRjbc0jjsqhsxPvfgrUrbIvKjEFLfAPvvLvcQIMUJf09SEjaaMkUAYd+
|
||||
TJaxFn5kd9Q6HbkD/fEN+lKhNZI40IJvfu7u4emUj3uKy9zrw576/T8aDYUl/own
|
||||
tqqfXh/jN8wnKCQwma7gaPmMOMqBt6zCsrN9/eKnMBpdULkUtjJD4NDg03XUFLlM
|
||||
am/oQ+MnasCcctkaXKbTGx3WfBVmkGj4b3Au18CVZkRWN2QsMdBC8JLRTICKse8U
|
||||
Mjybr/hQK3mnVdE=
|
||||
MIIDRTCCAi2gAwIBAgIUGoISfeW3LwSWHC52ORXdZY9pNLswDQYJKoZIhvcNAQEL
|
||||
BQAwMjELMAkGA1UEBhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9j
|
||||
YWxob3N0MB4XDTIwMDYyODEyNTQxNVoXDTIxMDYyODEyNTQxNVowMjELMAkGA1UE
|
||||
BhMCSUUxDzANBgNVBAoMBlNjcmFweTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjAN
|
||||
BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvCLxfTEQuIdf8JhiHrbVkGHYrNSK
|
||||
2XD2TCPaSIpJ2KKlFUrIz3A9tWlOfLnWabS5od89yOebhYj4DN/Qm2TViGg1mtWe
|
||||
pD1K2YWd1Af+hhAw5D+TpW2RH9TVhX7Ey5osWcl+0uy+RlKZE8qum72xi1vxWOmH
|
||||
wYw06iN8klQ3JfP2/eLRXBQjsh7WW0dbJ7yLvG6UFz1RbhFTtlxeIMenzNsHaMg7
|
||||
56Ru57/MMbaBwdBttXVzJDQ7imo8njuxDMszliC/QgIdBUBFzA2LB5qpr+v+laDN
|
||||
cN9t9Q9stsu446dFnRoofxJjMFW7lLu6h/lwP5r0kfeUkMDhXJ4mb6KwfwIDAQAB
|
||||
o1MwUTAdBgNVHQ4EFgQUVEdXn8ha2FA73zcy1Ia0FQMzMEYwHwYDVR0jBBgwFoAU
|
||||
VEdXn8ha2FA73zcy1Ia0FQMzMEYwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0B
|
||||
AQsFAAOCAQEAZpGBPsexMD+IwcMNIgc7FiaJsb8E30C9vWxgdnkpapi9zLJ4yiHQ
|
||||
VxkV9RTezUEADkaDj+2qFveamWTzJLnphgaaUpVeMcYACPhRVOYXidNrZyTmHIsX
|
||||
FwaTzAggW6CP7JxAcpxH0f9+NWFCZI36FihRdwuWyvrUl7rsXaexu0SOI/Ck0oWf
|
||||
2IW+jo67TSmcbte+J8wq77DX32mVLb/2nqpItH4T2Di+XjVBARACVOSdgdlo7lZE
|
||||
W8mSEXqP2BVx8JGG8X1znNLHcmjVj4EtkpH0wkYzpC4cvGkTsUcU7CU7ZyVUp+Bb
|
||||
dPMVxyRKWfAjRJc8o5Ot1mgHrx5coOtzAA==
|
||||
-----END CERTIFICATE-----
|
||||
|
|
|
|||
|
|
@ -1,28 +1,28 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCtY3MJSfm6a+t5
|
||||
ZitZkpPN53bhftuxD01N0crXqDzNiqP0amZ6/5kdvVOj7uNmLoS7wVLu60MRovq4
|
||||
JMJ6OpTBT+MuYJcUcoCJT7D+FkDt2J45srFn2MUKCJpdgPrYi3sKzGaHByNnDYKz
|
||||
vBXdLuyjIi5b1AjdKVcjsGf8pvaz2Dm36rhv8WExc4GA/8MtZ+AFxOL3t7mT3wd2
|
||||
RrQd31JrqMIOSmdS97lqb4sqq9agf/DV3DkS3oAD7KriwNOYJwzzrjLO3UjHijKY
|
||||
b6W0nq0VyxsSyH1jmdK7PKFfekXudoEhIP5CCEvrbN+7CaH3BJ8fQuBZE0fM14p/
|
||||
XmYYAmLNAgMBAAECggEAQKY4GlqO1seugRFrUHaqzbdkSCf42kgOVtnGfCqqoSj0
|
||||
gQm7NFlhSglxykokV9E4hJlMxvDJjSXrvgVWziRRmtKiroQtUN5wtsIUCGlbxFNk
|
||||
i7bpFwNoVJlolTymS1+WfSxBfk9XD/GlrkaPEG2SpjD0gCDLPUtQxmncHARVMDDu
|
||||
Eysk3njGghsTF7XMh8ljTE3CqqNSx9BkeWQr6EYfXcgaQ2jp9E+FspB5+KWeO4ss
|
||||
ELVHgtwmYSRPAEuz4XHz87RLuakqafko6ftvh3upVQwm0VXuwM+lEUYZrzoU2JQ4
|
||||
hePKHRaWQC4tawV6FyVHK4X0MuKP4uESr7YHbJ03sQKBgQDV4CyQU6xccW6hMxlD
|
||||
7hvrGcPQEPg6M4rX2uqWpB6RCh6stZEydYeh5S+A6ltml/2csw9Bl8nZM6KbArZa
|
||||
EKrZcOn7JgFyPpiDHqgEIx+9XL/mnsKMSkBKTFcvucVgjIWE8GT7jfAqMkcSysWf
|
||||
uRyUvtNpshmRLcdNhEjrr3vcwwKBgQDPid6sxBVcoyvrYUsRRVpXATJ9tsmU93LG
|
||||
HMHDlXkZ2CMfEuA0xLK+B9iyHMhh8NwYFjcG5oeVyVjE8SbifX4Sg49hde8ykXSR
|
||||
UBSNt22/JaWgreL95LEC/y9q+G4osli7NwRW1x6tB5cN1mE0hZI8Z0ETvyr3DoWO
|
||||
j/dbdFYJLwKBgDjVLCJiCbA6+EHfuTwC3upXW2BD0iJtJdz8MFA9Zl32SXZtfRri
|
||||
fls38qqYHBekFeF493nfouSTwwbb7qb6PNwxFAwH6mR4W8Cj+dO3nayNI/VdhKcQ
|
||||
6AqWRKjK/bcNQEG2O69Y5VPhLl/BAEjUQNMJ7lXs3LxmZMqld1cht5FPAoGBAJbI
|
||||
xXbiU97lUmCGZKLcr4EtBoEdz6GiksnrVMAEFmM3jHTkIu9TxcWZL9BgZxn5g/8g
|
||||
DMS/styZ2BvmVWkS4gkTepXFuI8V7Qoyk2xPS7Yn5QkzrQroH89clhfy/R4mTZ9f
|
||||
npB1ZP0z2YSdMCyXqyKlpjtxlga/jzt/z6irgmLTAoGAPrmudajtSBq534Ql2lPM
|
||||
8U6baRSAMMzV7MXcR8F1CRewQiYOzlgsB8toELNtjg1IGPqmoiNDDKmkHs3R2mO6
|
||||
J45kDPLFe9DTyZLZj0pWWK6yRLc/BA/gGzKFpMkNcyzLlQjNPqY/9mrrYea4J9Cj
|
||||
Z+pMCFLbwAbFZ9Qb/NFlUv0=
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC8IvF9MRC4h1/w
|
||||
mGIettWQYdis1IrZcPZMI9pIiknYoqUVSsjPcD21aU58udZptLmh3z3I55uFiPgM
|
||||
39CbZNWIaDWa1Z6kPUrZhZ3UB/6GEDDkP5OlbZEf1NWFfsTLmixZyX7S7L5GUpkT
|
||||
yq6bvbGLW/FY6YfBjDTqI3ySVDcl8/b94tFcFCOyHtZbR1snvIu8bpQXPVFuEVO2
|
||||
XF4gx6fM2wdoyDvnpG7nv8wxtoHB0G21dXMkNDuKajyeO7EMyzOWIL9CAh0FQEXM
|
||||
DYsHmqmv6/6VoM1w3231D2y2y7jjp0WdGih/EmMwVbuUu7qH+XA/mvSR95SQwOFc
|
||||
niZvorB/AgMBAAECggEAHVpSVRb/pdqxNEeCH4qlHWa2uJhcpXpDYzPAzcqNpPgT
|
||||
S5QkaoD3j8NDVKBl/I4O3FuJNzwzfo0VLmUJFgWQbzzbCDJGExfhArkfG8K3ilEi
|
||||
X6ovrgK/PrklKzPRHncKbmPKnrwDH9OpQHZB8diRx81rhVTCModehh1NRUNQa2I1
|
||||
QzFC7uyXx3duoIsI5QXVeEGuwHZfqIY/z+9SscdVFL6elXTPFUzBzcmAqQgdgWKN
|
||||
HXgX22LE0rAu8NnRvOZZWt4/nOjvlCFCPTB11NgthmKlVnsx4H7gpQ2OPh4bZ+0W
|
||||
birVEtZ3E1jxoGvw1FzxyqqpGkcanRMa8QWzK4JwuQKBgQDrgclpkqZrgHB/TC1p
|
||||
hLvsdflGI2SGs+c/mYR3GEjf0kJtI88WL5fj1QezdkDyOpwxFvnLslswfzdtzvis
|
||||
vksGysV35vhMPQUcmWhvzA7Pdxdv4BZr+ckER0SAYBBxg9KYZyxewGb5XzB8Cz2o
|
||||
8V+YpwrMAOYGuXHTfafv4CKlTQKBgQDMgetvV9/E3HNtKsATiPIwT3e1MzyPXigq
|
||||
12NkHSZa6s4yqm/h/fSUn54sJbhx+OtRRhktOo0aB34tcogtrJyClvCPdRAP/4Qi
|
||||
M43FjKo2cWiubWvtWlOZU04bpClG324q420rK7dCA2stID/Fa0sMQgAAyPH8TGMo
|
||||
gbvyrk4W+wKBgQDMIOnYZTF0epaH8BponJFaqwMOhTzr+OGW4dTMebMotZG4EdK8
|
||||
kzIfW5XaOsSecKjTb+vCYGzkA1CjEEPBTwuu7nDstblAM5/Lozi/tmqb7sjUwrIM
|
||||
kyxmVfONJjb6fV07lioCUtiui5B15DRkzBqlMRyNqLW43GJKA19d7rN4/QKBgCzy
|
||||
kRBTu/bEjQn9T2H7w18i2CiXLkREaYeg91NVpMxutwsjspt0+YCA5H7He5ZxIycl
|
||||
xPrP15tU8kKC3bNMMMny6sRc8j7R5fSuaAZ3OCHnIx7TJdlw9NbKHGyu0/Ojv87l
|
||||
VWUbopd7sN6mK930CvaSuvVxNN5C27hXazuXW8ppAoGBANcWsenNKpCJgF0cNPHX
|
||||
abPaWfcs5FKMNz8gEdGk3B1z/KBpYz59smPwurYVCXaWE6iv99sDOP7CVneF02sV
|
||||
SqyNzVhcVSG788uB3CwnpEvm7ydoH89L5dvYekAHP8RJulhWCK45lXkHLiYGKvhv
|
||||
PWuPk5VX+qF78JhUhPO3nfnu
|
||||
-----END PRIVATE KEY-----
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue