mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' of https://github.com/scrapy/scrapy into h2-client-protocol
This commit is contained in:
Aditya
commit
d97cf973dd
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.2.0
|
||||
current_version = 2.3.0
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
tests/sample_data/** binary
|
||||
16
.travis.yml
16
.travis.yml
|
|
@ -19,19 +19,25 @@ matrix:
|
|||
- env: TOXENV=typing
|
||||
python: 3.8
|
||||
|
||||
- env: TOXENV=pypy3
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5.2
|
||||
- env: TOXENV=asyncio
|
||||
- env: TOXENV=asyncio-pinned
|
||||
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
|
||||
- env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.7
|
||||
|
||||
- env: TOXENV=py PYPI_RELEASE_JOB=true
|
||||
python: 3.8
|
||||
dist: bionic
|
||||
|
|
@ -43,9 +49,9 @@ matrix:
|
|||
dist: bionic
|
||||
install:
|
||||
- |
|
||||
if [ "$TOXENV" = "pypy3" ]; then
|
||||
export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"
|
||||
wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
variables:
|
||||
TOXENV: py
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
strategy:
|
||||
matrix:
|
||||
Python35:
|
||||
python.version: '3.5'
|
||||
TOXENV: windows-pinned
|
||||
Python36:
|
||||
python.version: '3.6'
|
||||
Python37:
|
||||
python.version: '3.7'
|
||||
Python38:
|
||||
python.version: '3.8'
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
displayName: 'Use Python $(python.version)'
|
||||
- script: |
|
||||
pip install -U tox twine wheel codecov
|
||||
tox
|
||||
displayName: 'Run test suite'
|
||||
|
|
@ -17,7 +17,7 @@ class SettingsListDirective(Directive):
|
|||
def is_setting_index(node):
|
||||
if node.tagname == 'index':
|
||||
# index entries for setting directives look like:
|
||||
# [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
|
||||
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
|
||||
entry_type, info, refid = node['entries'][0][:3]
|
||||
return entry_type == 'pair' and info.endswith('; setting')
|
||||
return False
|
||||
|
|
|
|||
13
docs/conf.py
13
docs/conf.py
|
|
@ -284,6 +284,7 @@ intersphinx_mapping = {
|
|||
'attrs': ('https://www.attrs.org/en/stable/', None),
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
|
||||
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
'python': ('https://docs.python.org/3', None),
|
||||
'sphinx': ('https://www.sphinx-doc.org/en/master', None),
|
||||
|
|
@ -305,3 +306,15 @@ hoverxref_role_types = {
|
|||
"ref": "tooltip",
|
||||
}
|
||||
hoverxref_roles = ['command', 'reqmeta', 'setting', 'signal']
|
||||
|
||||
|
||||
def setup(app):
|
||||
app.connect('autodoc-skip-member', maybe_skip_member)
|
||||
|
||||
|
||||
def maybe_skip_member(app, what, name, obj, skip, options):
|
||||
if not skip:
|
||||
# autodocs was generating a text "alias of" for the following members
|
||||
# https://github.com/sphinx-doc/sphinx/issues/4422
|
||||
return name in {'default_item_class', 'default_selector_class'}
|
||||
return skip
|
||||
|
|
|
|||
|
|
@ -108,6 +108,11 @@ Well-written patches should:
|
|||
|
||||
tox -e docs-coverage
|
||||
|
||||
* if you are removing deprecated code, first make sure that at least 1 year
|
||||
(12 months) has passed since the release that introduced the deprecation.
|
||||
See :ref:`deprecation-policy`.
|
||||
|
||||
|
||||
.. _submitting-patches:
|
||||
|
||||
Submitting patches
|
||||
|
|
|
|||
14
docs/faq.rst
14
docs/faq.rst
|
|
@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
|
|||
|
||||
.. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use
|
||||
|
||||
.. _faq-python-versions:
|
||||
|
||||
What Python versions does Scrapy support?
|
||||
-----------------------------------------
|
||||
|
||||
Scrapy is supported under Python 3.5.2+
|
||||
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
|
||||
Python 3 support was added in Scrapy 1.1.
|
||||
PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.
|
||||
Python 2 support was dropped in Scrapy 2.0.
|
||||
|
||||
.. note::
|
||||
For Python 3 support on Windows, it is recommended to use
|
||||
Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.
|
||||
|
||||
Did Scrapy "steal" X from Django?
|
||||
---------------------------------
|
||||
|
|
|
|||
|
|
@ -4,12 +4,18 @@
|
|||
Installation guide
|
||||
==================
|
||||
|
||||
.. _faq-python-versions:
|
||||
|
||||
Supported Python versions
|
||||
=========================
|
||||
|
||||
Scrapy requires Python 3.5.2+, either the CPython implementation (default) or
|
||||
the PyPy 5.9+ implementation (see :ref:`python:implementations`).
|
||||
|
||||
|
||||
Installing Scrapy
|
||||
=================
|
||||
|
||||
Scrapy runs on Python 3.5.2 or above under CPython (default Python
|
||||
implementation) and PyPy (starting with PyPy 5.9).
|
||||
|
||||
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
|
||||
the `conda-forge`_ channel, which has up-to-date packages for Linux, Windows
|
||||
and macOS.
|
||||
|
|
|
|||
133
docs/news.rst
133
docs/news.rst
|
|
@ -3,6 +3,139 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.3.0:
|
||||
|
||||
Scrapy 2.3.0 (2020-08-04)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* :ref:`Feed exports <topics-feed-exports>` now support :ref:`Google Cloud
|
||||
Storage <topics-feed-storage-gcs>` as a storage backend
|
||||
|
||||
* The new :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting allows to deliver
|
||||
output items in batches of up to the specified number of items.
|
||||
|
||||
It also serves as a workaround for :ref:`delayed file delivery
|
||||
<delayed-file-delivery>`, which causes Scrapy to only start item delivery
|
||||
after the crawl has finished when using certain storage backends
|
||||
(:ref:`S3 <topics-feed-storage-s3>`, :ref:`FTP <topics-feed-storage-ftp>`,
|
||||
and now :ref:`GCS <topics-feed-storage-gcs>`).
|
||||
|
||||
* The base implementation of :ref:`item loaders <topics-loaders>` has been
|
||||
moved into a separate library, :doc:`itemloaders <itemloaders:index>`,
|
||||
allowing usage from outside Scrapy and a separate release schedule
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Removed the following classes and their parent modules from
|
||||
``scrapy.linkextractors``:
|
||||
|
||||
* ``htmlparser.HtmlParserLinkExtractor``
|
||||
* ``regex.RegexLinkExtractor``
|
||||
* ``sgml.BaseSgmlLinkExtractor``
|
||||
* ``sgml.SgmlLinkExtractor``
|
||||
|
||||
Use
|
||||
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`
|
||||
instead (:issue:`4356`, :issue:`4679`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* The ``scrapy.utils.python.retry_on_eintr`` function is now deprecated
|
||||
(:issue:`4683`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* :ref:`Feed exports <topics-feed-exports>` support :ref:`Google Cloud
|
||||
Storage <topics-feed-storage-gcs>` (:issue:`685`, :issue:`3608`)
|
||||
|
||||
* New :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` setting for batch deliveries
|
||||
(:issue:`4250`, :issue:`4434`)
|
||||
|
||||
* The :command:`parse` command now allows specifying an output file
|
||||
(:issue:`4317`, :issue:`4377`)
|
||||
|
||||
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
|
||||
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now also support
|
||||
``--data-raw`` (:issue:`4612`)
|
||||
|
||||
* A ``parse`` callback may now be used in built-in spider subclasses, such
|
||||
as :class:`~scrapy.spiders.CrawlSpider` (:issue:`712`, :issue:`732`,
|
||||
:issue:`781`, :issue:`4254` )
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* Fixed the :ref:`CSV exporting <topics-feed-format-csv>` of
|
||||
:ref:`dataclass items <dataclass-items>` and :ref:`attr.s items
|
||||
<attrs-items>` (:issue:`4667`, :issue:`4668`)
|
||||
|
||||
* :meth:`Request.from_curl <scrapy.http.Request.from_curl>` and
|
||||
:func:`~scrapy.utils.curl.curl_to_request_kwargs` now set the request
|
||||
method to ``POST`` when a request body is specified and no request method
|
||||
is specified (:issue:`4612`)
|
||||
|
||||
* The processing of ANSI escape sequences in enabled in Windows 10.0.14393
|
||||
and later, where it is required for colored output (:issue:`4393`,
|
||||
:issue:`4403`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* Updated the `OpenSSL cipher list format`_ link in the documentation about
|
||||
the :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` setting (:issue:`4653`)
|
||||
|
||||
* Simplified the code example in :ref:`topics-loaders-dataclass`
|
||||
(:issue:`4652`)
|
||||
|
||||
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
* The base implementation of :ref:`item loaders <topics-loaders>` has been
|
||||
moved into :doc:`itemloaders <itemloaders:index>` (:issue:`4005`,
|
||||
:issue:`4516`)
|
||||
|
||||
* Fixed a silenced error in some scheduler tests (:issue:`4644`,
|
||||
:issue:`4645`)
|
||||
|
||||
* Renewed the localhost certificate used for SSL tests (:issue:`4650`)
|
||||
|
||||
* Removed cookie-handling code specific to Python 2 (:issue:`4682`)
|
||||
|
||||
* Stopped using Python 2 unicode literal syntax (:issue:`4704`)
|
||||
|
||||
* Stopped using a backlash for line continuation (:issue:`4673`)
|
||||
|
||||
* Removed unneeded entries from the MyPy exception list (:issue:`4690`)
|
||||
|
||||
* Automated tests now pass on Windows as part of our continuous integration
|
||||
system (:issue:`4458`)
|
||||
|
||||
* Automated tests now pass on the latest PyPy version for supported Python
|
||||
versions in our continuous integration system (:issue:`4504`)
|
||||
|
||||
|
||||
.. _release-2.2.1:
|
||||
|
||||
Scrapy 2.2.1 (2020-07-17)
|
||||
-------------------------
|
||||
|
||||
* The :command:`startproject` command no longer makes unintended changes to
|
||||
the permissions of files in the destination folder, such as removing
|
||||
execution permissions (:issue:`4662`, :issue:`4666`)
|
||||
|
||||
|
||||
.. _release-2.2.0:
|
||||
|
||||
Scrapy 2.2.0 (2020-06-24)
|
||||
|
|
|
|||
|
|
@ -468,7 +468,7 @@ Supported options:
|
|||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||
response
|
||||
|
||||
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
|
||||
* ``--meta`` or ``-m``: additional request meta that will be passed to the callback
|
||||
request. This must be a valid json string. Example: --meta='{"foo" : "bar"}'
|
||||
|
||||
* ``--cbkwargs``: additional keyword arguments that will be passed to the callback.
|
||||
|
|
@ -491,6 +491,10 @@ Supported options:
|
|||
|
||||
* ``--verbose`` or ``-v``: display information for each depth level
|
||||
|
||||
* ``--output`` or ``-o``: dump scraped items to a file
|
||||
|
||||
.. versionadded:: 2.3
|
||||
|
||||
.. skip: start
|
||||
|
||||
Usage example::
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ override three methods:
|
|||
.. class:: Contract(method, *args)
|
||||
|
||||
:param method: callback function to which the contract is associated
|
||||
:type method: function
|
||||
:type method: collections.abc.Callable
|
||||
|
||||
:param args: list of arguments passed into the docstring (whitespace
|
||||
separated)
|
||||
|
|
|
|||
|
|
@ -289,8 +289,10 @@ request::
|
|||
"://quotes.toscrape.com/scroll' -H 'Cache-Control: max-age=0'")
|
||||
|
||||
Alternatively, if you want to know the arguments needed to recreate that
|
||||
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
|
||||
function to get a dictionary with the equivalent arguments.
|
||||
request you can use the :func:`~scrapy.utils.curl.curl_to_request_kwargs`
|
||||
function to get a dictionary with the equivalent arguments:
|
||||
|
||||
.. autofunction:: scrapy.utils.curl.curl_to_request_kwargs
|
||||
|
||||
Note that to translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
|
|
|||
|
|
@ -62,10 +62,10 @@ rest of the framework.
|
|||
:type smtpport: int
|
||||
|
||||
:param smtptls: enforce using SMTP STARTTLS
|
||||
:type smtptls: boolean
|
||||
:type smtptls: bool
|
||||
|
||||
:param smtpssl: enforce using a secure SSL connection
|
||||
:type smtpssl: boolean
|
||||
:type smtpssl: bool
|
||||
|
||||
.. classmethod:: from_settings(settings)
|
||||
|
||||
|
|
@ -79,14 +79,14 @@ rest of the framework.
|
|||
|
||||
Send email to the given recipients.
|
||||
|
||||
:param to: the e-mail recipients
|
||||
:type to: str or list of str
|
||||
:param to: the e-mail recipients as a string or as a list of strings
|
||||
:type to: str or list
|
||||
|
||||
:param subject: the subject of the e-mail
|
||||
:type subject: str
|
||||
|
||||
:param cc: the e-mails to CC
|
||||
:type cc: str or list of str
|
||||
:param cc: the e-mails to CC as a string or as a list of strings
|
||||
:type cc: str or list
|
||||
|
||||
:param body: the e-mail body
|
||||
:type body: str
|
||||
|
|
@ -96,7 +96,7 @@ rest of the framework.
|
|||
appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
|
||||
attachment and ``file_object`` is a readable file object with the
|
||||
contents of the attachment
|
||||
:type attachs: iterable
|
||||
:type attachs: collections.abc.Iterable
|
||||
|
||||
:param mimetype: the MIME type of the e-mail
|
||||
:type mimetype: str
|
||||
|
|
|
|||
|
|
@ -166,8 +166,7 @@ BaseItemExporter
|
|||
By default, this method looks for a serializer :ref:`declared in the item
|
||||
field <topics-exporters-serializers>` and returns the result of applying
|
||||
that serializer to the value. If no serializer is found, it returns the
|
||||
value unchanged except for ``unicode`` values which are encoded to
|
||||
``str`` using the encoding declared in the :attr:`encoding` attribute.
|
||||
value unchanged.
|
||||
|
||||
:param field: the field being serialized. If the source :ref:`item object
|
||||
<item-types>` does not define field metadata, *field* is an empty
|
||||
|
|
@ -217,10 +216,7 @@ BaseItemExporter
|
|||
|
||||
.. attribute:: encoding
|
||||
|
||||
The encoding that will be used to encode unicode values. This only
|
||||
affects unicode values (which are always serialized to str using this
|
||||
encoding). Other value types are passed unchanged to the specific
|
||||
serialization library.
|
||||
The output character encoding.
|
||||
|
||||
.. attribute:: indent
|
||||
|
||||
|
|
@ -309,7 +305,7 @@ CsvItemExporter
|
|||
:param include_headers_line: If enabled, makes the exporter output a header
|
||||
line with the field names taken from
|
||||
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
|
||||
:type include_headers_line: boolean
|
||||
:type include_headers_line: bool
|
||||
|
||||
:param join_multivalued: The char (or chars) that will be used for joining
|
||||
multi-valued fields, if found.
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ The storages backends supported out of the box are:
|
|||
* :ref:`topics-feed-storage-fs`
|
||||
* :ref:`topics-feed-storage-ftp`
|
||||
* :ref:`topics-feed-storage-s3` (requires botocore_)
|
||||
* :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
|
||||
* :ref:`topics-feed-storage-stdout`
|
||||
|
||||
Some storage backends may be unavailable if the required external libraries are
|
||||
|
|
@ -169,6 +170,9 @@ FTP supports two different connection modes: `active or passive
|
|||
mode by default. To use the active connection mode instead, set the
|
||||
:setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
|
||||
.. _topics-feed-storage-s3:
|
||||
|
||||
S3
|
||||
|
|
@ -194,6 +198,37 @@ You can also define a custom ACL for exported feeds using this setting:
|
|||
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
|
||||
.. _topics-feed-storage-gcs:
|
||||
|
||||
Google Cloud Storage (GCS)
|
||||
--------------------------
|
||||
|
||||
.. versionadded:: 2.3
|
||||
|
||||
The feeds are stored on `Google Cloud Storage`_.
|
||||
|
||||
* URI scheme: ``gs``
|
||||
* Example URIs:
|
||||
|
||||
* ``gs://mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `google-cloud-storage`_.
|
||||
|
||||
For more information about authentication, please refer to `Google Cloud documentation <https://cloud.google.com/docs/authentication/production>`_.
|
||||
|
||||
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
|
||||
|
||||
* :setting:`FEED_STORAGE_GCS_ACL`
|
||||
* :setting:`GCS_PROJECT_ID`
|
||||
|
||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||
|
||||
.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
|
||||
|
||||
|
||||
.. _topics-feed-storage-stdout:
|
||||
|
||||
Standard output
|
||||
|
|
@ -206,6 +241,26 @@ The feeds are written to the standard output of the Scrapy process.
|
|||
* Required external libraries: none
|
||||
|
||||
|
||||
.. _delayed-file-delivery:
|
||||
|
||||
Delayed file delivery
|
||||
---------------------
|
||||
|
||||
As indicated above, some of the described storage backends use delayed file
|
||||
delivery.
|
||||
|
||||
These storage backends do not upload items to the feed URI as those items are
|
||||
scraped. Instead, Scrapy writes items into a temporary local file, and only
|
||||
once all the file contents have been written (i.e. at the end of the crawl) is
|
||||
that file uploaded to the feed URI.
|
||||
|
||||
If you want item delivery to start earlier when using one of these storage
|
||||
backends, use :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` to split the output items
|
||||
in multiple files, with the specified maximum item count per file. That way, as
|
||||
soon as a file reaches the maximum item count, that file is delivered to the
|
||||
feed URI, allowing item delivery to start way before the end of the crawl.
|
||||
|
||||
|
||||
Settings
|
||||
========
|
||||
|
||||
|
|
@ -220,6 +275,7 @@ These are the settings used for configuring the feed exports:
|
|||
* :setting:`FEED_STORAGE_FTP_ACTIVE`
|
||||
* :setting:`FEED_STORAGE_S3_ACL`
|
||||
* :setting:`FEED_EXPORTERS`
|
||||
* :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
|
||||
.. currentmodule:: scrapy.extensions.feedexport
|
||||
|
||||
|
|
@ -265,12 +321,14 @@ The following is a list of the accepted keys and the setting that is used
|
|||
as a fallback value if that key is not provided for a specific feed definition.
|
||||
|
||||
* ``format``: the serialization format to be used for the feed.
|
||||
See :ref:`topics-feed-format` for possible values.
|
||||
See :ref:`topics-feed-format` for possible values.
|
||||
Mandatory, no fallback setting
|
||||
* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
|
||||
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
|
||||
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
|
||||
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
|
||||
* ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -425,7 +483,111 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
'csv': None,
|
||||
}
|
||||
|
||||
|
||||
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
|
||||
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT
|
||||
-----------------------------
|
||||
|
||||
Default: ``0``
|
||||
|
||||
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
|
||||
storing up to the specified number of items in each output file.
|
||||
|
||||
When generating multiple output files, you must use at least one of the following
|
||||
placeholders in the feed URI to indicate how the different output file names are
|
||||
generated:
|
||||
|
||||
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
|
||||
(e.g. ``2020-03-28T14-45-08.237134``)
|
||||
|
||||
* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
|
||||
|
||||
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
|
||||
alter the number format. For example, to make the batch ID a 5-digit
|
||||
number by introducing leading zeroes as needed, use ``%(batch_id)05d``
|
||||
(e.g. ``3`` becomes ``00003``, ``123`` becomes ``00123``).
|
||||
|
||||
For instance, if your settings include::
|
||||
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT = 100
|
||||
|
||||
And your :command:`crawl` command line is::
|
||||
|
||||
scrapy crawl spidername -o "dirname/%(batch_id)d-filename%(batch_time)s.json"
|
||||
|
||||
The command line above can generate a directory tree like::
|
||||
|
||||
->projectname
|
||||
-->dirname
|
||||
--->1-filename2020-03-28T14-45-08.237134.json
|
||||
--->2-filename2020-03-28T14-45-09.148903.json
|
||||
--->3-filename2020-03-28T14-45-10.046092.json
|
||||
|
||||
Where the first and second files contain exactly 100 items. The last one contains
|
||||
100 items or fewer.
|
||||
|
||||
|
||||
.. setting:: FEED_URI_PARAMS
|
||||
|
||||
FEED_URI_PARAMS
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
A string with the import path of a function to set the parameters to apply with
|
||||
:ref:`printf-style string formatting <python:old-string-formatting>` to the
|
||||
feed URI.
|
||||
|
||||
The function signature should be as follows:
|
||||
|
||||
.. function:: uri_params(params, spider)
|
||||
|
||||
Return a :class:`dict` of key-value pairs to apply to the feed URI using
|
||||
:ref:`printf-style string formatting <python:old-string-formatting>`.
|
||||
|
||||
:param params: default key-value pairs
|
||||
|
||||
Specifically:
|
||||
|
||||
- ``batch_id``: ID of the file batch. See
|
||||
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
|
||||
is always ``1``.
|
||||
|
||||
- ``batch_time``: UTC date and time, in ISO format with ``:``
|
||||
replaced with ``-``.
|
||||
|
||||
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
- ``time``: ``batch_time``, with microseconds set to ``0``.
|
||||
:type params: dict
|
||||
|
||||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.spiders.Spider
|
||||
|
||||
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
#. Define the following function somewhere in your project::
|
||||
|
||||
# myproject/utils.py
|
||||
def uri_params(params, spider):
|
||||
return {**params, 'spider_name': spider.name}
|
||||
|
||||
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
|
||||
|
||||
# myproject/settings.py
|
||||
FEED_URI_PARAMS = 'myproject.utils.uri_params'
|
||||
|
||||
#. Use ``%(spider_name)s`` in your feed URI::
|
||||
|
||||
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
|
||||
|
||||
|
||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
|
||||
:param ignore: if given, all objects from the specified class (or tuple of
|
||||
classes) will be ignored.
|
||||
:type ignore: class or classes tuple
|
||||
:type ignore: type or tuple
|
||||
|
||||
.. function:: get_oldest(class_name)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ LxmlLinkExtractor
|
|||
:param allow: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be extracted. If not
|
||||
given (or empty), it will match all links.
|
||||
:type allow: a regular expression (or list of)
|
||||
:type allow: str or list
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: a regular expression (or list of)
|
||||
:type deny: str or list
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
|
|
@ -88,7 +88,7 @@ LxmlLinkExtractor
|
|||
that the link's text must match in order to be extracted. If not
|
||||
given (or empty), it will match all links. If a list of regular expressions is
|
||||
given, the link will be extracted if it matches at least one.
|
||||
:type restrict_text: a regular expression (or list of)
|
||||
:type restrict_text: str or list
|
||||
|
||||
:param tags: a tag or a list of tags to consider when extracting links.
|
||||
Defaults to ``('a', 'area')``.
|
||||
|
|
@ -106,11 +106,11 @@ LxmlLinkExtractor
|
|||
different for requests with canonicalized and raw URLs. If you're
|
||||
using LinkExtractor to follow links it is more robust to
|
||||
keep the default ``canonicalize=False``.
|
||||
:type canonicalize: boolean
|
||||
:type canonicalize: bool
|
||||
|
||||
:param unique: whether duplicate filtering should be applied to extracted
|
||||
links.
|
||||
:type unique: boolean
|
||||
:type unique: bool
|
||||
|
||||
:param process_value: a function which receives each value extracted from
|
||||
the tag and attributes scanned and can modify the value and return a
|
||||
|
|
@ -132,7 +132,7 @@ LxmlLinkExtractor
|
|||
if m:
|
||||
return m.group(1)
|
||||
|
||||
:type process_value: callable
|
||||
:type process_value: collections.abc.Callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
|
|
@ -141,7 +141,7 @@ LxmlLinkExtractor
|
|||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: boolean
|
||||
:type strip: bool
|
||||
|
||||
.. automethod:: extract_links
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,10 @@ Item Loaders are designed to provide a flexible, efficient and easy mechanism
|
|||
for extending and overriding different field parsing rules, either by spider,
|
||||
or by source format (HTML, XML, etc) without becoming a nightmare to maintain.
|
||||
|
||||
.. note:: Item Loaders are an extension of the itemloaders_ library that make it
|
||||
easier to work with Scrapy by adding support for
|
||||
:ref:`responses <topics-request-response>`.
|
||||
|
||||
Using Item Loaders to populate items
|
||||
====================================
|
||||
|
||||
|
|
@ -88,29 +92,17 @@ item loaders: unless a pre-populated item is passed to the loader, fields
|
|||
will be populated incrementally using the loader's :meth:`~ItemLoader.add_xpath`,
|
||||
:meth:`~ItemLoader.add_css` and :meth:`~ItemLoader.add_value` methods.
|
||||
|
||||
Given the way that item loaders store data internally, one approach
|
||||
to overcome this is to define items using the :func:`~dataclasses.field`
|
||||
function, with ``list`` as the ``default_factory`` argument::
|
||||
One approach to overcome this is to define items using the
|
||||
:func:`~dataclasses.field` function, with a ``default`` argument::
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
@dataclass
|
||||
class InventoryItem:
|
||||
name: str = field(default_factory=list)
|
||||
price: float = field(default_factory=list)
|
||||
stock: int = field(default_factory=list)
|
||||
|
||||
Note that in order to keep the example simple, the types do not match
|
||||
completely. A more accurate but verbose definition would be::
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Union
|
||||
|
||||
@dataclass
|
||||
class InventoryItem:
|
||||
name: Union[str, List[str]] = field(default_factory=list)
|
||||
price: Union[float, List[float]] = field(default_factory=list)
|
||||
stock: Union[int, List[int]] = field(default_factory=list)
|
||||
name: Optional[str] = field(default=None)
|
||||
price: Optional[float] = field(default=None)
|
||||
stock: Optional[int] = field(default=None)
|
||||
|
||||
|
||||
.. _topics-loaders-processors:
|
||||
|
|
@ -185,8 +177,8 @@ The other thing you need to keep in mind is that the values returned by input
|
|||
processors are collected internally (in lists) and then passed to output
|
||||
processors to populate the fields.
|
||||
|
||||
Last, but not least, Scrapy comes with some :ref:`commonly used processors
|
||||
<topics-loaders-available-processors>` built-in for convenience.
|
||||
Last, but not least, itemloaders_ comes with some :ref:`commonly used
|
||||
processors <itemloaders:built-in-processors>` built-in for convenience.
|
||||
|
||||
|
||||
Declaring Item Loaders
|
||||
|
|
@ -194,17 +186,17 @@ Declaring Item Loaders
|
|||
|
||||
Item Loaders are declared using a class definition syntax. Here is an example::
|
||||
|
||||
from itemloaders.processors import TakeFirst, MapCompose, Join
|
||||
from scrapy.loader import ItemLoader
|
||||
from scrapy.loader.processors import TakeFirst, MapCompose, Join
|
||||
|
||||
class ProductLoader(ItemLoader):
|
||||
|
||||
default_output_processor = TakeFirst()
|
||||
|
||||
name_in = MapCompose(unicode.title)
|
||||
name_in = MapCompose(str.title)
|
||||
name_out = Join()
|
||||
|
||||
price_in = MapCompose(unicode.strip)
|
||||
price_in = MapCompose(str.strip)
|
||||
|
||||
# ...
|
||||
|
||||
|
|
@ -226,7 +218,7 @@ output processors to use: in the :ref:`Item Field <topics-items-fields>`
|
|||
metadata. Here is an example::
|
||||
|
||||
import scrapy
|
||||
from scrapy.loader.processors import Join, MapCompose, TakeFirst
|
||||
from itemloaders.processors import Join, MapCompose, TakeFirst
|
||||
from w3lib.html import remove_tags
|
||||
|
||||
def filter_price(value):
|
||||
|
|
@ -245,10 +237,10 @@ metadata. Here is an example::
|
|||
|
||||
>>> from scrapy.loader import ItemLoader
|
||||
>>> il = ItemLoader(item=Product())
|
||||
>>> il.add_value('name', [u'Welcome to my', u'<strong>website</strong>'])
|
||||
>>> il.add_value('price', [u'€', u'<span>1000</span>'])
|
||||
>>> il.add_value('name', ['Welcome to my', '<strong>website</strong>'])
|
||||
>>> il.add_value('price', ['€', '<span>1000</span>'])
|
||||
>>> il.load_item()
|
||||
{'name': u'Welcome to my website', 'price': u'1000'}
|
||||
{'name': 'Welcome to my website', 'price': '1000'}
|
||||
|
||||
The precedence order, for both input and output processors, is as follows:
|
||||
|
||||
|
|
@ -307,250 +299,9 @@ There are several ways to modify Item Loader context values:
|
|||
ItemLoader objects
|
||||
==================
|
||||
|
||||
.. class:: ItemLoader([item, selector, response], **kwargs)
|
||||
|
||||
Return a new Item Loader for populating the given :ref:`item object
|
||||
<topics-items>`. If no item object is given, one is instantiated
|
||||
automatically using the class in :attr:`default_item_class`.
|
||||
|
||||
When instantiated with a ``selector`` or a ``response`` parameters
|
||||
the :class:`ItemLoader` class provides convenient mechanisms for extracting
|
||||
data from web pages using :ref:`selectors <topics-selectors>`.
|
||||
|
||||
:param item: The item instance to populate using subsequent calls to
|
||||
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
|
||||
or :meth:`~ItemLoader.add_value`.
|
||||
:type item: :ref:`item object <topics-items>`
|
||||
|
||||
:param selector: The selector to extract data from, when using the
|
||||
:meth:`add_xpath` (resp. :meth:`add_css`) or :meth:`replace_xpath`
|
||||
(resp. :meth:`replace_css`) method.
|
||||
:type selector: :class:`~scrapy.selector.Selector` object
|
||||
|
||||
:param response: The response used to construct the selector using the
|
||||
:attr:`default_selector_class`, unless the selector argument is given,
|
||||
in which case this argument is ignored.
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
The item, selector, response and the remaining keyword arguments are
|
||||
assigned to the Loader context (accessible through the :attr:`context` attribute).
|
||||
|
||||
:class:`ItemLoader` instances have the following methods:
|
||||
|
||||
.. method:: get_value(value, *processors, **kwargs)
|
||||
|
||||
Process the given ``value`` by the given ``processors`` and keyword
|
||||
arguments.
|
||||
|
||||
Available keyword arguments:
|
||||
|
||||
:param re: a regular expression to use for extracting data from the
|
||||
given value using :meth:`~scrapy.utils.misc.extract_regex` method,
|
||||
applied before processors
|
||||
:type re: str or compiled regex
|
||||
|
||||
Examples:
|
||||
|
||||
>>> from scrapy.loader.processors import TakeFirst
|
||||
>>> loader.get_value(u'name: foo', TakeFirst(), unicode.upper, re='name: (.+)')
|
||||
'FOO`
|
||||
|
||||
.. method:: add_value(field_name, value, *processors, **kwargs)
|
||||
|
||||
Process and then add the given ``value`` for the given field.
|
||||
|
||||
The value is first passed through :meth:`get_value` by giving the
|
||||
``processors`` and ``kwargs``, and then passed through the
|
||||
:ref:`field input processor <topics-loaders-processors>` and its result
|
||||
appended to the data collected for that field. If the field already
|
||||
contains collected data, the new data is added.
|
||||
|
||||
The given ``field_name`` can be ``None``, in which case values for
|
||||
multiple fields may be added. And the processed value should be a dict
|
||||
with field_name mapped to values.
|
||||
|
||||
Examples::
|
||||
|
||||
loader.add_value('name', u'Color TV')
|
||||
loader.add_value('colours', [u'white', u'blue'])
|
||||
loader.add_value('length', u'100')
|
||||
loader.add_value('name', u'name: foo', TakeFirst(), re='name: (.+)')
|
||||
loader.add_value(None, {'name': u'foo', 'sex': u'male'})
|
||||
|
||||
.. method:: replace_value(field_name, value, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_value` but replaces the collected data with the
|
||||
new value instead of adding it.
|
||||
.. method:: get_xpath(xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.get_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of unicode strings from the
|
||||
selector associated with this :class:`ItemLoader`.
|
||||
|
||||
:param xpath: the XPath to extract data from
|
||||
:type xpath: str
|
||||
|
||||
:param re: a regular expression to use for extracting data from the
|
||||
selected XPath region
|
||||
:type re: str or compiled regex
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.get_xpath('//p[@class="product-name"]')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.get_xpath('//p[@id="price"]', TakeFirst(), re='the price is (.*)')
|
||||
|
||||
.. method:: add_xpath(field_name, xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.add_value` but receives an XPath instead of a
|
||||
value, which is used to extract a list of unicode strings from the
|
||||
selector associated with this :class:`ItemLoader`.
|
||||
|
||||
See :meth:`get_xpath` for ``kwargs``.
|
||||
|
||||
:param xpath: the XPath to extract data from
|
||||
:type xpath: str
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.add_xpath('name', '//p[@class="product-name"]')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.add_xpath('price', '//p[@id="price"]', re='the price is (.*)')
|
||||
|
||||
.. method:: replace_xpath(field_name, xpath, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_xpath` but replaces collected data instead of
|
||||
adding it.
|
||||
|
||||
.. method:: get_css(css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.get_value` but receives a CSS selector
|
||||
instead of a value, which is used to extract a list of unicode strings
|
||||
from the selector associated with this :class:`ItemLoader`.
|
||||
|
||||
:param css: the CSS selector to extract data from
|
||||
:type css: str
|
||||
|
||||
:param re: a regular expression to use for extracting data from the
|
||||
selected CSS region
|
||||
:type re: str or compiled regex
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.get_css('p.product-name')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.get_css('p#price', TakeFirst(), re='the price is (.*)')
|
||||
|
||||
.. method:: add_css(field_name, css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`ItemLoader.add_value` but receives a CSS selector
|
||||
instead of a value, which is used to extract a list of unicode strings
|
||||
from the selector associated with this :class:`ItemLoader`.
|
||||
|
||||
See :meth:`get_css` for ``kwargs``.
|
||||
|
||||
:param css: the CSS selector to extract data from
|
||||
:type css: str
|
||||
|
||||
Examples::
|
||||
|
||||
# HTML snippet: <p class="product-name">Color TV</p>
|
||||
loader.add_css('name', 'p.product-name')
|
||||
# HTML snippet: <p id="price">the price is $1200</p>
|
||||
loader.add_css('price', 'p#price', re='the price is (.*)')
|
||||
|
||||
.. method:: replace_css(field_name, css, *processors, **kwargs)
|
||||
|
||||
Similar to :meth:`add_css` but replaces collected data instead of
|
||||
adding it.
|
||||
|
||||
.. method:: load_item()
|
||||
|
||||
Populate the item with the data collected so far, and return it. The
|
||||
data collected is first passed through the :ref:`output processors
|
||||
<topics-loaders-processors>` to get the final value to assign to each
|
||||
item field.
|
||||
|
||||
.. method:: nested_xpath(xpath)
|
||||
|
||||
Create a nested loader with an xpath selector.
|
||||
The supplied selector is applied relative to selector associated
|
||||
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
|
||||
object <topics-items>` with the parent :class:`ItemLoader` so calls to
|
||||
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
|
||||
behave as expected.
|
||||
|
||||
.. method:: nested_css(css)
|
||||
|
||||
Create a nested loader with a css selector.
|
||||
The supplied selector is applied relative to selector associated
|
||||
with this :class:`ItemLoader`. The nested loader shares the :ref:`item
|
||||
object <topics-items>` with the parent :class:`ItemLoader` so calls to
|
||||
:meth:`add_xpath`, :meth:`add_value`, :meth:`replace_value`, etc. will
|
||||
behave as expected.
|
||||
|
||||
.. method:: get_collected_values(field_name)
|
||||
|
||||
Return the collected values for the given field.
|
||||
|
||||
.. method:: get_output_value(field_name)
|
||||
|
||||
Return the collected values parsed using the output processor, for the
|
||||
given field. This method doesn't populate or modify the item at all.
|
||||
|
||||
.. method:: get_input_processor(field_name)
|
||||
|
||||
Return the input processor for the given field.
|
||||
|
||||
.. method:: get_output_processor(field_name)
|
||||
|
||||
Return the output processor for the given field.
|
||||
|
||||
:class:`ItemLoader` instances have the following attributes:
|
||||
|
||||
.. attribute:: item
|
||||
|
||||
The :ref:`item object <topics-items>` being parsed by this Item Loader.
|
||||
This is mostly used as a property so when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
The currently active :ref:`Context <topics-loaders-context>` of this
|
||||
Item Loader.
|
||||
|
||||
.. attribute:: default_item_class
|
||||
|
||||
An :ref:`item object <topics-items>` class or factory, used to
|
||||
instantiate items when not given in the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
The default input processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_output_processor
|
||||
|
||||
The default output processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
.. autoclass:: scrapy.loader.ItemLoader
|
||||
:members:
|
||||
:inherited-members:
|
||||
|
||||
.. _topics-loaders-nested:
|
||||
|
||||
|
|
@ -621,7 +372,7 @@ those dashes in the final product names.
|
|||
Here's how you can remove those dashes by reusing and extending the default
|
||||
Product Item Loader (``ProductLoader``)::
|
||||
|
||||
from scrapy.loader.processors import MapCompose
|
||||
from itemloaders.processors import MapCompose
|
||||
from myproject.ItemLoaders import ProductLoader
|
||||
|
||||
def strip_dashes(x):
|
||||
|
|
@ -634,7 +385,7 @@ Another case where extending Item Loaders can be very helpful is when you have
|
|||
multiple source formats, for example XML and HTML. In the XML version you may
|
||||
want to remove ``CDATA`` occurrences. Here's an example of how to do it::
|
||||
|
||||
from scrapy.loader.processors import MapCompose
|
||||
from itemloaders.processors import MapCompose
|
||||
from myproject.ItemLoaders import ProductLoader
|
||||
from myproject.utils.xml import remove_cdata
|
||||
|
||||
|
|
@ -654,156 +405,5 @@ projects. Scrapy only provides the mechanism; it doesn't impose any specific
|
|||
organization of your Loaders collection - that's up to you and your project's
|
||||
needs.
|
||||
|
||||
.. _topics-loaders-available-processors:
|
||||
|
||||
Available built-in processors
|
||||
=============================
|
||||
|
||||
.. module:: scrapy.loader.processors
|
||||
:synopsis: A collection of processors to use with Item Loaders
|
||||
|
||||
Even though you can use any callable function as input and output processors,
|
||||
Scrapy provides some commonly used processors, which are described below. Some
|
||||
of them, like the :class:`MapCompose` (which is typically used as input
|
||||
processor) compose the output of several functions executed in order, to
|
||||
produce the final parsed value.
|
||||
|
||||
Here is a list of all built-in processors:
|
||||
|
||||
.. class:: Identity
|
||||
|
||||
The simplest processor, which doesn't do anything. It returns the original
|
||||
values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
|
||||
accept Loader contexts.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import Identity
|
||||
>>> proc = Identity()
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
['one', 'two', 'three']
|
||||
|
||||
.. class:: TakeFirst
|
||||
|
||||
Returns the first non-null/non-empty value from the values received,
|
||||
so it's typically used as an output processor to single-valued fields.
|
||||
It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import TakeFirst
|
||||
>>> proc = TakeFirst()
|
||||
>>> proc(['', 'one', 'two', 'three'])
|
||||
'one'
|
||||
|
||||
.. class:: Join(separator=u' ')
|
||||
|
||||
Returns the values joined with the separator given in the ``__init__`` method, which
|
||||
defaults to ``u' '``. It doesn't accept Loader contexts.
|
||||
|
||||
When using the default separator, this processor is equivalent to the
|
||||
function: ``u' '.join``
|
||||
|
||||
Examples:
|
||||
|
||||
>>> from scrapy.loader.processors import Join
|
||||
>>> proc = Join()
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
'one two three'
|
||||
>>> proc = Join('<br>')
|
||||
>>> proc(['one', 'two', 'three'])
|
||||
'one<br>two<br>three'
|
||||
|
||||
.. class:: Compose(*functions, **default_loader_context)
|
||||
|
||||
A processor which is constructed from the composition of the given
|
||||
functions. This means that each input value of this processor is passed to
|
||||
the first function, and the result of that function is passed to the second
|
||||
function, and so on, until the last function returns the output value of
|
||||
this processor.
|
||||
|
||||
By default, stop process on ``None`` value. This behaviour can be changed by
|
||||
passing keyword argument ``stop_on_none=False``.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import Compose
|
||||
>>> proc = Compose(lambda v: v[0], str.upper)
|
||||
>>> proc(['hello', 'world'])
|
||||
'HELLO'
|
||||
|
||||
Each function can optionally receive a ``loader_context`` parameter. For
|
||||
those which do, this processor will pass the currently active :ref:`Loader
|
||||
context <topics-loaders-context>` through that parameter.
|
||||
|
||||
The keyword arguments passed in the ``__init__`` method are used as the default
|
||||
Loader context values passed to each function call. However, the final
|
||||
Loader context values passed to functions are overridden with the currently
|
||||
active Loader context accessible through the :meth:`ItemLoader.context`
|
||||
attribute.
|
||||
|
||||
.. class:: MapCompose(*functions, **default_loader_context)
|
||||
|
||||
A processor which is constructed from the composition of the given
|
||||
functions, similar to the :class:`Compose` processor. The difference with
|
||||
this processor is the way internal results are passed among functions,
|
||||
which is as follows:
|
||||
|
||||
The input value of this processor is *iterated* and the first function is
|
||||
applied to each element. The results of these function calls (one for each element)
|
||||
are concatenated to construct a new iterable, which is then used to apply the
|
||||
second function, and so on, until the last function is applied to each
|
||||
value of the list of values collected so far. The output values of the last
|
||||
function are concatenated together to produce the output of this processor.
|
||||
|
||||
Each particular function can return a value or a list of values, which is
|
||||
flattened with the list of values returned by the same function applied to
|
||||
the other input values. The functions can also return ``None`` in which
|
||||
case the output of that function is ignored for further processing over the
|
||||
chain.
|
||||
|
||||
This processor provides a convenient way to compose functions that only
|
||||
work with single values (instead of iterables). For this reason the
|
||||
:class:`MapCompose` processor is typically used as input processor, since
|
||||
data is often extracted using the
|
||||
:meth:`~scrapy.selector.Selector.extract` method of :ref:`selectors
|
||||
<topics-selectors>`, which returns a list of unicode strings.
|
||||
|
||||
The example below should clarify how it works:
|
||||
|
||||
>>> def filter_world(x):
|
||||
... return None if x == 'world' else x
|
||||
...
|
||||
>>> from scrapy.loader.processors import MapCompose
|
||||
>>> proc = MapCompose(filter_world, str.upper)
|
||||
>>> proc(['hello', 'world', 'this', 'is', 'scrapy'])
|
||||
['HELLO, 'THIS', 'IS', 'SCRAPY']
|
||||
|
||||
As with the Compose processor, functions can receive Loader contexts, and
|
||||
``__init__`` method keyword arguments are used as default context values. See
|
||||
:class:`Compose` processor for more info.
|
||||
|
||||
.. class:: SelectJmes(json_path)
|
||||
|
||||
Queries the value using the json path provided to the ``__init__`` method and returns the output.
|
||||
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
|
||||
This processor takes only one input at a time.
|
||||
|
||||
Example:
|
||||
|
||||
>>> from scrapy.loader.processors import SelectJmes, Compose, MapCompose
|
||||
>>> proc = SelectJmes("foo") #for direct use on lists and dictionaries
|
||||
>>> proc({'foo': 'bar'})
|
||||
'bar'
|
||||
>>> proc({'foo': {'bar': 'baz'}})
|
||||
{'bar': 'baz'}
|
||||
|
||||
Working with Json:
|
||||
|
||||
>>> import json
|
||||
>>> proc_single_json_str = Compose(json.loads, SelectJmes("foo"))
|
||||
>>> proc_single_json_str('{"foo": "bar"}')
|
||||
'bar'
|
||||
>>> proc_json_list = Compose(json.loads, MapCompose(SelectJmes('foo')))
|
||||
>>> proc_json_list('[{"foo":"bar"}, {"baz":"tar"}]')
|
||||
['bar']
|
||||
.. _itemloaders: https://itemloaders.readthedocs.io/en/latest/
|
||||
.. _processors: https://itemloaders.readthedocs.io/en/latest/built-in-processors.html
|
||||
|
|
|
|||
|
|
@ -207,7 +207,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
|
|||
Google Cloud Storage
|
||||
---------------------
|
||||
|
||||
.. setting:: GCS_PROJECT_ID
|
||||
.. setting:: FILES_STORE_GCS_ACL
|
||||
.. setting:: IMAGES_STORE_GCS_ACL
|
||||
|
||||
|
|
@ -413,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
.. class:: FilesPipeline
|
||||
|
||||
.. method:: file_path(self, request, response=None, info=None)
|
||||
.. method:: file_path(self, request, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called once per downloaded item. It returns the
|
||||
download path of the file originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>` and
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
|
||||
:class:`request <scrapy.Request>`,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.item.Item>`
|
||||
|
||||
You can override this method to customize the download path of each file.
|
||||
|
||||
|
|
@ -437,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
class MyFilesPipeline(FilesPipeline):
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return 'files/' + os.path.basename(urlparse(request.url).path)
|
||||
|
||||
Similarly, you can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`file_path` method returns
|
||||
``full/<request URL hash>.<extension>``.
|
||||
|
||||
|
|
@ -545,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
|
||||
customizing the field names and adding custom behavior for images.
|
||||
|
||||
.. method:: file_path(self, request, response=None, info=None)
|
||||
.. method:: file_path(self, request, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called once per downloaded item. It returns the
|
||||
download path of the file originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>` and
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
|
||||
:class:`request <scrapy.Request>`,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.item.Item>`
|
||||
|
||||
You can override this method to customize the download path of each file.
|
||||
|
||||
|
|
@ -569,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
|
||||
class MyImagesPipeline(ImagesPipeline):
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return 'files/' + os.path.basename(urlparse(request.url).path)
|
||||
|
||||
Similarly, you can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`file_path` method returns
|
||||
``full/<request URL hash>.<extension>``.
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Request objects
|
|||
:param url: the URL of this request
|
||||
|
||||
If the URL is invalid, a :exc:`ValueError` exception is raised.
|
||||
:type url: string
|
||||
:type url: str
|
||||
|
||||
:param callback: the function that will be called with the response of this
|
||||
request (once it's downloaded) as its first parameter. For more information
|
||||
|
|
@ -42,21 +42,21 @@ Request objects
|
|||
:meth:`~scrapy.spiders.Spider.parse` method will be used.
|
||||
Note that if exceptions are raised during processing, errback is called instead.
|
||||
|
||||
:type callback: callable
|
||||
:type callback: collections.abc.Callable
|
||||
|
||||
:param method: the HTTP method of this request. Defaults to ``'GET'``.
|
||||
:type method: string
|
||||
:type method: str
|
||||
|
||||
:param meta: the initial values for the :attr:`Request.meta` attribute. If
|
||||
given, the dict passed in this parameter will be shallow copied.
|
||||
:type meta: dict
|
||||
|
||||
:param body: the request body. If a ``unicode`` is passed, then it's encoded to
|
||||
``str`` using the ``encoding`` passed (which defaults to ``utf-8``). If
|
||||
``body`` is not given, an empty string is stored. Regardless of the
|
||||
type of this argument, the final value stored will be a ``str`` (never
|
||||
``unicode`` or ``None``).
|
||||
:type body: str or unicode
|
||||
:param body: the request body. If a string is passed, then it's encoded as
|
||||
bytes using the ``encoding`` passed (which defaults to ``utf-8``). If
|
||||
``body`` is not given, an empty bytes object is stored. Regardless of the
|
||||
type of this argument, the final value stored will be a bytes object
|
||||
(never a string or ``None``).
|
||||
:type body: bytes or str
|
||||
|
||||
:param headers: the headers of this request. The dict values can be strings
|
||||
(for single valued headers) or lists (for multi-valued headers). If
|
||||
|
|
@ -106,8 +106,8 @@ Request objects
|
|||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
This encoding will be used to percent-encode the URL and to convert the
|
||||
body to ``str`` (if given as ``unicode``).
|
||||
:type encoding: string
|
||||
body to bytes (if given as a string).
|
||||
:type encoding: str
|
||||
|
||||
:param priority: the priority of this request (defaults to ``0``).
|
||||
The priority is used by the scheduler to define the order used to process
|
||||
|
|
@ -119,7 +119,7 @@ Request objects
|
|||
the scheduler. This is used when you want to perform an identical
|
||||
request multiple times, to ignore the duplicates filter. Use it with
|
||||
care, or you will get into crawling loops. Default to ``False``.
|
||||
:type dont_filter: boolean
|
||||
:type dont_filter: bool
|
||||
|
||||
:param errback: a function that will be called if any exception was
|
||||
raised while processing the request. This includes pages that failed
|
||||
|
|
@ -131,7 +131,7 @@ Request objects
|
|||
.. versionchanged:: 2.0
|
||||
The *callback* parameter is no longer required when the *errback*
|
||||
parameter is specified.
|
||||
:type errback: callable
|
||||
:type errback: collections.abc.Callable
|
||||
|
||||
:param flags: Flags sent to the request, can be used for logging or similar purposes.
|
||||
:type flags: list
|
||||
|
|
@ -159,7 +159,7 @@ Request objects
|
|||
|
||||
.. attribute:: Request.body
|
||||
|
||||
A str that contains the request body.
|
||||
The request body as bytes.
|
||||
|
||||
This attribute is read-only. To change the body of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -485,7 +485,7 @@ fields with form data from :class:`Response` objects.
|
|||
:param formdata: is a dictionary (or iterable of (key, value) tuples)
|
||||
containing HTML Form data which will be url-encoded and assigned to the
|
||||
body of the request.
|
||||
:type formdata: dict or iterable of tuples
|
||||
:type formdata: dict or collections.abc.Iterable
|
||||
|
||||
The :class:`FormRequest` objects support the following class method in
|
||||
addition to the standard :class:`Request` methods:
|
||||
|
|
@ -517,20 +517,20 @@ fields with form data from :class:`Response` objects.
|
|||
:type response: :class:`Response` object
|
||||
|
||||
:param formname: if given, the form with name attribute set to this value will be used.
|
||||
:type formname: string
|
||||
:type formname: str
|
||||
|
||||
:param formid: if given, the form with id attribute set to this value will be used.
|
||||
:type formid: string
|
||||
:type formid: str
|
||||
|
||||
:param formxpath: if given, the first form that matches the xpath will be used.
|
||||
:type formxpath: string
|
||||
:type formxpath: str
|
||||
|
||||
:param formcss: if given, the first form that matches the css selector will be used.
|
||||
:type formcss: string
|
||||
:type formcss: str
|
||||
|
||||
:param formnumber: the number of form to use, when the response contains
|
||||
multiple forms. The first one (and also the default) is ``0``.
|
||||
:type formnumber: integer
|
||||
:type formnumber: int
|
||||
|
||||
:param formdata: fields to override in the form data. If a field was
|
||||
already present in the response ``<form>`` element, its value is
|
||||
|
|
@ -548,7 +548,7 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
:param dont_click: If True, the form data will be submitted without
|
||||
clicking in any element.
|
||||
:type dont_click: boolean
|
||||
:type dont_click: bool
|
||||
|
||||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` ``__init__`` method.
|
||||
|
|
@ -636,7 +636,7 @@ dealing with JSON requests.
|
|||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
:type data: object
|
||||
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
|
||||
data into JSON format.
|
||||
|
|
@ -663,16 +663,16 @@ Response objects
|
|||
downloaded (by the Downloader) and fed to the Spiders for processing.
|
||||
|
||||
:param url: the URL of this response
|
||||
:type url: string
|
||||
:type url: str
|
||||
|
||||
:param status: the HTTP status of the response. Defaults to ``200``.
|
||||
:type status: integer
|
||||
:type status: int
|
||||
|
||||
:param headers: the headers of this response. The dict values can be strings
|
||||
(for single valued headers) or lists (for multi-valued headers).
|
||||
:type headers: dict
|
||||
|
||||
:param body: the response body. To access the decoded text as str you can use
|
||||
:param body: the response body. To access the decoded text as a string, use
|
||||
``response.text`` from an encoding-aware
|
||||
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
|
||||
such as :class:`TextResponse`.
|
||||
|
|
@ -720,10 +720,10 @@ Response objects
|
|||
|
||||
.. attribute:: Response.body
|
||||
|
||||
The body of this Response. Keep in mind that Response.body
|
||||
is always a bytes object. If you want the unicode version use
|
||||
:attr:`TextResponse.text` (only available in :class:`TextResponse`
|
||||
and subclasses).
|
||||
The response body as bytes.
|
||||
|
||||
If you want the body as a string, use :attr:`TextResponse.text` (only
|
||||
available in :class:`TextResponse` and subclasses).
|
||||
|
||||
This attribute is read-only. To change the body of a Response use
|
||||
:meth:`replace`.
|
||||
|
|
@ -842,18 +842,18 @@ TextResponse objects
|
|||
is the same as for the :class:`Response` class and is not documented here.
|
||||
|
||||
:param encoding: is a string which contains the encoding to use for this
|
||||
response. If you create a :class:`TextResponse` object with a unicode
|
||||
body, it will be encoded using this encoding (remember the body attribute
|
||||
is always a string). If ``encoding`` is ``None`` (default value), the
|
||||
encoding will be looked up in the response headers and body instead.
|
||||
:type encoding: string
|
||||
response. If you create a :class:`TextResponse` object with a string as
|
||||
body, it will be converted to bytes encoded using this encoding. If
|
||||
*encoding* is ``None`` (default), the encoding will be looked up in the
|
||||
response headers and body instead.
|
||||
:type encoding: str
|
||||
|
||||
:class:`TextResponse` objects support the following attributes in addition
|
||||
to the standard :class:`Response` ones:
|
||||
|
||||
.. attribute:: TextResponse.text
|
||||
|
||||
Response body, as unicode.
|
||||
Response body, as a string.
|
||||
|
||||
The same as ``response.body.decode(response.encoding)``, but the
|
||||
result is cached after the first call, so you can access
|
||||
|
|
@ -861,9 +861,11 @@ TextResponse objects
|
|||
|
||||
.. note::
|
||||
|
||||
``unicode(response.body)`` is not a correct way to convert response
|
||||
body to unicode: you would be using the system default encoding
|
||||
(typically ``ascii``) instead of the response encoding.
|
||||
``str(response.body)`` is not a correct way to convert the response
|
||||
body into a string:
|
||||
|
||||
>>> str(b'body')
|
||||
"b'body'"
|
||||
|
||||
|
||||
.. attribute:: TextResponse.encoding
|
||||
|
|
|
|||
|
|
@ -64,7 +64,8 @@ more shortcuts: ``response.xpath()`` and ``response.css()``:
|
|||
|
||||
Scrapy selectors are instances of :class:`~scrapy.selector.Selector` class
|
||||
constructed by passing either :class:`~scrapy.http.TextResponse` object or
|
||||
markup as an unicode string (in ``text`` argument).
|
||||
markup as a string (in ``text`` argument).
|
||||
|
||||
Usually there is no need to construct Scrapy selectors manually:
|
||||
``response`` object is available in Spider callbacks, so in most cases
|
||||
it is more convenient to use ``response.css()`` and ``response.xpath()``
|
||||
|
|
@ -383,7 +384,7 @@ Using selectors with regular expressions
|
|||
|
||||
:class:`~scrapy.selector.Selector` also has a ``.re()`` method for extracting
|
||||
data using regular expressions. However, unlike using ``.xpath()`` or
|
||||
``.css()`` methods, ``.re()`` returns a list of unicode strings. So you
|
||||
``.css()`` methods, ``.re()`` returns a list of strings. So you
|
||||
can't construct nested ``.re()`` calls.
|
||||
|
||||
Here's an example used to extract image names from the :ref:`HTML code
|
||||
|
|
@ -734,7 +735,7 @@ The ``test()`` function, for example, can prove quite useful when XPath's
|
|||
Example selecting links in list item with a "class" attribute ending with a digit:
|
||||
|
||||
>>> from scrapy import Selector
|
||||
>>> doc = u"""
|
||||
>>> doc = """
|
||||
... <div>
|
||||
... <ul>
|
||||
... <li class="item-0"><a href="link1.html">first item</a></li>
|
||||
|
|
@ -765,7 +766,7 @@ extracting text elements for example.
|
|||
Example extracting microdata (sample content taken from https://schema.org/Product)
|
||||
with groups of itemscopes and corresponding itemprops::
|
||||
|
||||
>>> doc = u"""
|
||||
>>> doc = """
|
||||
... <div itemscope itemtype="http://schema.org/Product">
|
||||
... <span itemprop="name">Kenmore White 17" Microwave</span>
|
||||
... <img src="kenmore-microwave-17in.jpg" alt='Kenmore 17" Microwave' />
|
||||
|
|
@ -989,7 +990,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
|
|||
sel.xpath("//h1")
|
||||
|
||||
2. Extract the text of all ``<h1>`` elements from an HTML response body,
|
||||
returning a list of unicode strings::
|
||||
returning a list of strings::
|
||||
|
||||
sel.xpath("//h1").getall() # this includes the h1 tag
|
||||
sel.xpath("//h1/text()").getall() # this excludes the h1 tag
|
||||
|
|
|
|||
|
|
@ -469,7 +469,7 @@ necessary to access certain HTTPS websites: for example, you may need to use
|
|||
``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
|
||||
specific cipher that is not included in ``DEFAULT`` if a website requires it.
|
||||
|
||||
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/ciphers.html#CIPHER-LIST-FORMAT
|
||||
.. _OpenSSL cipher list format: https://www.openssl.org/docs/manmaster/man1/openssl-ciphers.html#CIPHER-LIST-FORMAT
|
||||
|
||||
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
|
||||
|
||||
|
|
@ -802,6 +802,14 @@ The Feed Temp dir allows you to set a custom folder to save crawler
|
|||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||
|
||||
.. setting:: FEED_STORAGE_GCS_ACL
|
||||
|
||||
FEED_STORAGE_GCS_ACL
|
||||
--------------------
|
||||
|
||||
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
||||
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://cloud.google.com/storage/docs/access-control/lists>`_.
|
||||
|
||||
.. setting:: FTP_PASSIVE_MODE
|
||||
|
||||
FTP_PASSIVE_MODE
|
||||
|
|
@ -841,6 +849,15 @@ Default: ``"anonymous"``
|
|||
The username to use for FTP connections when there is no ``"ftp_user"``
|
||||
in ``Request`` meta.
|
||||
|
||||
.. setting:: GCS_PROJECT_ID
|
||||
|
||||
GCS_PROJECT_ID
|
||||
-----------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
The Project ID that will be used when storing data on `Google Cloud Storage`_.
|
||||
|
||||
.. setting:: ITEM_PIPELINES
|
||||
|
||||
ITEM_PIPELINES
|
||||
|
|
@ -1560,3 +1577,4 @@ case to see how to enable and use them.
|
|||
.. _Amazon web services: https://aws.amazon.com/
|
||||
.. _breadth-first order: https://en.wikipedia.org/wiki/Breadth-first_search
|
||||
.. _depth-first order: https://en.wikipedia.org/wiki/Depth-first_search
|
||||
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
||||
|
|
|
|||
|
|
@ -360,9 +360,10 @@ CrawlSpider
|
|||
|
||||
This spider also exposes an overrideable method:
|
||||
|
||||
.. method:: parse_start_url(response)
|
||||
.. method:: parse_start_url(response, **kwargs)
|
||||
|
||||
This method is called for the start_urls responses. It allows to parse
|
||||
This method is called for each response produced for the URLs in
|
||||
the spider's ``start_urls`` attribute. It allows to parse
|
||||
the initial responses and must return either an
|
||||
:ref:`item object <topics-items>`, a :class:`~scrapy.http.Request`
|
||||
object, or an iterable containing any of them.
|
||||
|
|
@ -388,11 +389,6 @@ Crawling rules
|
|||
object will contain the text of the link that produced the :class:`~scrapy.http.Request`
|
||||
in its ``meta`` dictionary (under the ``link_text`` key)
|
||||
|
||||
.. warning:: When writing crawl spider rules, avoid using ``parse`` as
|
||||
callback, since the :class:`CrawlSpider` uses the ``parse`` method
|
||||
itself to implement its logic. So if you override the ``parse`` method,
|
||||
the crawl spider will no longer work.
|
||||
|
||||
``cb_kwargs`` is a dict containing the keyword arguments to be passed to the
|
||||
callback function.
|
||||
|
||||
|
|
@ -418,6 +414,11 @@ Crawling rules
|
|||
It receives a :class:`Twisted Failure <twisted.python.failure.Failure>`
|
||||
instance as first parameter.
|
||||
|
||||
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`CrawlSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
|
||||
.. versionadded:: 2.0
|
||||
The *errback* parameter.
|
||||
|
||||
|
|
@ -451,6 +452,11 @@ Let's now take a look at an example CrawlSpider with rules::
|
|||
item['name'] = response.xpath('//td[@id="item_name"]/text()').get()
|
||||
item['description'] = response.xpath('//td[@id="item_description"]/text()').get()
|
||||
item['link_text'] = response.meta['link_text']
|
||||
url = response.xpath('//td[@id="additional_data"]/@href').get()
|
||||
return response.follow(url, self.parse_additional_page, cb_kwargs=dict(item=item))
|
||||
|
||||
def parse_additional_page(self, response, item):
|
||||
item['additional_data'] = response.xpath('//p[@id="additional_data"]/text()').get()
|
||||
return item
|
||||
|
||||
|
||||
|
|
@ -544,6 +550,11 @@ XMLFeedSpider
|
|||
those results. It must return a list of results (items or requests).
|
||||
|
||||
|
||||
.. warning:: Because of its internal implementation, you must explicitly set
|
||||
callbacks for new requests when writing :class:`XMLFeedSpider`-based spiders;
|
||||
unexpected behaviour can occur otherwise.
|
||||
|
||||
|
||||
XMLFeedSpider example
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ def main():
|
|||
_contents = None
|
||||
|
||||
# A regex that matches standard linkcheck output lines
|
||||
line_re = re.compile(u'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
line_re = re.compile(r'(.*)\:\d+\:\s\[(.*)\]\s(?:(.*)\sto\s(.*)|(.*))')
|
||||
|
||||
# Read lines from the linkcheck output file
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
.. _versioning:
|
||||
|
||||
============================
|
||||
Versioning and API Stability
|
||||
Versioning and API stability
|
||||
============================
|
||||
|
||||
Versioning
|
||||
|
|
@ -34,7 +34,7 @@ For example:
|
|||
production)
|
||||
|
||||
|
||||
API Stability
|
||||
API stability
|
||||
=============
|
||||
|
||||
API stability was one of the major goals for the *1.0* release.
|
||||
|
|
@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
|
|||
same way.
|
||||
|
||||
|
||||
.. _deprecation-policy:
|
||||
|
||||
Deprecation policy
|
||||
==================
|
||||
|
||||
We aim to maintain support for deprecated Scrapy features for at least 1 year.
|
||||
|
||||
For example, if a feature is deprecated in a Scrapy version released on
|
||||
June 15th 2020, that feature should continue to work in versions released on
|
||||
June 14th 2021 or before that.
|
||||
|
||||
Any new Scrapy release after a year *may* remove support for that deprecated
|
||||
feature.
|
||||
|
||||
All deprecated features removed in a Scrapy release are explicitly mentioned in
|
||||
the :ref:`release notes <news>`.
|
||||
|
||||
|
||||
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
|
||||
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ class QPSSpider(Spider):
|
|||
slots = 1
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(QPSSpider, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
if self.qps is not None:
|
||||
self.qps = float(self.qps)
|
||||
self.download_delay = 1 / self.qps
|
||||
|
|
|
|||
|
|
@ -40,3 +40,4 @@ flake8-ignore =
|
|||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.2.0
|
||||
2.3.0
|
||||
|
|
|
|||
|
|
@ -19,10 +19,12 @@ def _iter_command_classes(module_name):
|
|||
# scrapy.utils.spider.iter_spider_classes
|
||||
for module in walk_modules(module_name):
|
||||
for obj in vars(module).values():
|
||||
if inspect.isclass(obj) and \
|
||||
issubclass(obj, ScrapyCommand) and \
|
||||
obj.__module__ == module.__name__ and \
|
||||
not obj == ScrapyCommand:
|
||||
if (
|
||||
inspect.isclass(obj)
|
||||
and issubclass(obj, ScrapyCommand)
|
||||
and obj.__module__ == module.__name__
|
||||
and not obj == ScrapyCommand
|
||||
):
|
||||
yield obj
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ class ScrapyCommand:
|
|||
|
||||
class BaseRunSpiderCommand(ScrapyCommand):
|
||||
"""
|
||||
Common class used to share functionality between the crawl and runspider commands
|
||||
Common class used to share functionality between the crawl, parse and runspider commands
|
||||
"""
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
|
|
|
|||
|
|
@ -78,19 +78,19 @@ class Command(ScrapyCommand):
|
|||
elif tested_methods:
|
||||
self.crawler_process.crawl(spidercls)
|
||||
|
||||
# start checks
|
||||
if opts.list:
|
||||
for spider, methods in sorted(contract_reqs.items()):
|
||||
if not methods and not opts.verbose:
|
||||
continue
|
||||
print(spider)
|
||||
for method in sorted(methods):
|
||||
print(' * %s' % method)
|
||||
else:
|
||||
start = time.time()
|
||||
self.crawler_process.start()
|
||||
stop = time.time()
|
||||
# start checks
|
||||
if opts.list:
|
||||
for spider, methods in sorted(contract_reqs.items()):
|
||||
if not methods and not opts.verbose:
|
||||
continue
|
||||
print(spider)
|
||||
for method in sorted(methods):
|
||||
print(' * %s' % method)
|
||||
else:
|
||||
start = time.time()
|
||||
self.crawler_process.start()
|
||||
stop = time.time()
|
||||
|
||||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
result.printErrors()
|
||||
result.printSummary(start, stop)
|
||||
self.exitcode = int(not result.wasSuccessful())
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ class Command(BaseRunSpiderCommand):
|
|||
else:
|
||||
self.crawler_process.start()
|
||||
|
||||
if self.crawler_process.bootstrap_failed or \
|
||||
(hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception):
|
||||
if (
|
||||
self.crawler_process.bootstrap_failed
|
||||
or hasattr(self.crawler_process, 'has_exception') and self.crawler_process.has_exception
|
||||
):
|
||||
self.exitcode = 1
|
||||
|
|
|
|||
|
|
@ -19,8 +19,10 @@ class Command(ScrapyCommand):
|
|||
return "Fetch a URL using the Scrapy downloader"
|
||||
|
||||
def long_desc(self):
|
||||
return "Fetch a URL using the Scrapy downloader and print its content " \
|
||||
"to stdout. You may want to use --nolog to disable logging"
|
||||
return (
|
||||
"Fetch a URL using the Scrapy downloader and print its content"
|
||||
" to stdout. You may want to use --nolog to disable logging"
|
||||
)
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
|
|
|
|||
|
|
@ -121,6 +121,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
@property
|
||||
def templates_dir(self):
|
||||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'spiders')
|
||||
return join(
|
||||
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
|
||||
'spiders'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,18 +4,16 @@ import logging
|
|||
from itemadapter import is_item, ItemAdapter
|
||||
from w3lib.url import is_url
|
||||
|
||||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.commands import BaseRunSpiderCommand
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils import display
|
||||
from scrapy.utils.conf import arglist_to_dict
|
||||
from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
|
||||
from scrapy.exceptions import UsageError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
||||
class Command(BaseRunSpiderCommand):
|
||||
requires_project = True
|
||||
|
||||
spider = None
|
||||
|
|
@ -31,11 +29,9 @@ class Command(ScrapyCommand):
|
|||
return "Parse URL (using its spider) and print the results"
|
||||
|
||||
def add_options(self, parser):
|
||||
ScrapyCommand.add_options(self, parser)
|
||||
BaseRunSpiderCommand.add_options(self, parser)
|
||||
parser.add_option("--spider", dest="spider", default=None,
|
||||
help="use this spider without looking for one")
|
||||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("--pipelines", action="store_true",
|
||||
help="process items through pipelines")
|
||||
parser.add_option("--nolinks", dest="nolinks", action="store_true",
|
||||
|
|
@ -200,12 +196,15 @@ class Command(ScrapyCommand):
|
|||
self.add_items(depth, items)
|
||||
self.add_requests(depth, requests)
|
||||
|
||||
scraped_data = items if opts.output else []
|
||||
if depth < opts.depth:
|
||||
for req in requests:
|
||||
req.meta['_depth'] = depth + 1
|
||||
req.meta['_callback'] = req.callback
|
||||
req.callback = callback
|
||||
return requests
|
||||
scraped_data += requests
|
||||
|
||||
return scraped_data
|
||||
|
||||
# update request meta if any extra meta was passed through the --meta/-m opts.
|
||||
if opts.meta:
|
||||
|
|
@ -221,18 +220,11 @@ class Command(ScrapyCommand):
|
|||
return request
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
BaseRunSpiderCommand.process_options(self, args, opts)
|
||||
|
||||
self.process_spider_arguments(opts)
|
||||
self.process_request_meta(opts)
|
||||
self.process_request_cb_kwargs(opts)
|
||||
|
||||
def process_spider_arguments(self, opts):
|
||||
try:
|
||||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
|
||||
def process_request_meta(self, opts):
|
||||
if opts.meta:
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import re
|
||||
import os
|
||||
import stat
|
||||
import string
|
||||
from importlib import import_module
|
||||
from os.path import join, exists, abspath
|
||||
from shutil import ignore_patterns, move, copy2, copystat
|
||||
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
|
||||
|
||||
import scrapy
|
||||
from scrapy.commands import ScrapyCommand
|
||||
|
|
@ -20,7 +20,12 @@ TEMPLATES_TO_RENDER = (
|
|||
('${project_name}', 'middlewares.py.tmpl'),
|
||||
)
|
||||
|
||||
IGNORE = ignore_patterns('*.pyc', '.svn')
|
||||
IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
|
||||
|
||||
|
||||
def _make_writable(path):
|
||||
current_permissions = os.stat(path).st_mode
|
||||
os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
|
||||
|
||||
|
||||
class Command(ScrapyCommand):
|
||||
|
|
@ -78,30 +83,10 @@ class Command(ScrapyCommand):
|
|||
self._copytree(srcname, dstname)
|
||||
else:
|
||||
copy2(srcname, dstname)
|
||||
_make_writable(dstname)
|
||||
|
||||
copystat(src, dst)
|
||||
self._set_rw_permissions(dst)
|
||||
|
||||
def _set_rw_permissions(self, path):
|
||||
"""
|
||||
Sets permissions of a directory tree to +rw and +rwx for folders.
|
||||
This is necessary if the start template files come without write
|
||||
permissions.
|
||||
"""
|
||||
mode_rw = (stat.S_IRUSR
|
||||
| stat.S_IWUSR
|
||||
| stat.S_IRGRP
|
||||
| stat.S_IROTH)
|
||||
|
||||
mode_x = (stat.S_IXUSR
|
||||
| stat.S_IXGRP
|
||||
| stat.S_IXOTH)
|
||||
|
||||
os.chmod(path, mode_rw | mode_x)
|
||||
for root, dirs, files in os.walk(path):
|
||||
for dir in dirs:
|
||||
os.chmod(join(root, dir), mode_rw | mode_x)
|
||||
for file in files:
|
||||
os.chmod(join(root, file), mode_rw)
|
||||
_make_writable(dst)
|
||||
|
||||
def run(self, args, opts):
|
||||
if len(args) not in (1, 2):
|
||||
|
|
@ -137,6 +122,7 @@ class Command(ScrapyCommand):
|
|||
|
||||
@property
|
||||
def templates_dir(self):
|
||||
_templates_base_dir = self.settings['TEMPLATES_DIR'] or \
|
||||
join(scrapy.__path__[0], 'templates')
|
||||
return join(_templates_base_dir, 'project')
|
||||
return join(
|
||||
self.settings['TEMPLATES_DIR'] or join(scrapy.__path__[0], 'templates'),
|
||||
'project'
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,11 +8,10 @@ class Command(fetch.Command):
|
|||
return "Open URL in browser, as seen by Scrapy"
|
||||
|
||||
def long_desc(self):
|
||||
return "Fetch a URL using the Scrapy downloader and show its " \
|
||||
"contents in a browser"
|
||||
return "Fetch a URL using the Scrapy downloader and show its contents in a browser"
|
||||
|
||||
def add_options(self, parser):
|
||||
super(Command, self).add_options(parser)
|
||||
super().add_options(parser)
|
||||
parser.remove_option("--headers")
|
||||
|
||||
def _print_response(self, response, opts):
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ class ReturnsContract(Contract):
|
|||
}
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(ReturnsContract, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
if len(self.args) not in [1, 2, 3]:
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
"""
|
||||
|
||||
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
|
||||
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self._ssl_method = method
|
||||
self.tls_verbose_logging = tls_verbose_logging
|
||||
if tls_ciphers:
|
||||
|
|
@ -48,7 +48,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
|||
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
|
||||
#
|
||||
# * getattr() for `_ssl_method` attribute for context factories
|
||||
# not calling super(..., self).__init__
|
||||
# not calling super().__init__
|
||||
return CertificateOptions(
|
||||
verify=False,
|
||||
method=getattr(self, 'method', getattr(self, '_ssl_method', None)),
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
|
||||
def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None):
|
||||
proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
|
||||
super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
|
||||
super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
|
||||
self._tunnelReadyDeferred = defer.Deferred()
|
||||
self._tunneledHost = host
|
||||
self._tunneledPort = port
|
||||
|
|
@ -155,7 +155,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
|
|||
|
||||
def connect(self, protocolFactory):
|
||||
self._protocolFactory = protocolFactory
|
||||
connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory)
|
||||
connectDeferred = super().connect(protocolFactory)
|
||||
connectDeferred.addCallback(self.requestTunnel)
|
||||
connectDeferred.addErrback(self.connectFailed)
|
||||
return self._tunnelReadyDeferred
|
||||
|
|
@ -192,7 +192,7 @@ class TunnelingAgent(Agent):
|
|||
|
||||
def __init__(self, reactor, proxyConf, contextFactory=None,
|
||||
connectTimeout=None, bindAddress=None, pool=None):
|
||||
super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
|
||||
super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
|
||||
self._proxyConf = proxyConf
|
||||
self._contextFactory = contextFactory
|
||||
|
||||
|
|
@ -212,7 +212,7 @@ class TunnelingAgent(Agent):
|
|||
# otherwise, same remote host connection request could reuse
|
||||
# a cached tunneled connection to a different proxy
|
||||
key = key + self._proxyConf
|
||||
return super(TunnelingAgent, self)._requestWithEndpoint(
|
||||
return super()._requestWithEndpoint(
|
||||
key=key,
|
||||
endpoint=endpoint,
|
||||
method=method,
|
||||
|
|
@ -226,7 +226,7 @@ class TunnelingAgent(Agent):
|
|||
class ScrapyProxyAgent(Agent):
|
||||
|
||||
def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None):
|
||||
super(ScrapyProxyAgent, self).__init__(
|
||||
super().__init__(
|
||||
reactor=reactor,
|
||||
connectTimeout=connectTimeout,
|
||||
bindAddress=bindAddress,
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
"""
|
||||
|
||||
def __init__(self, hostname, ctx, verbose_logging=False):
|
||||
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
|
||||
super().__init__(hostname, ctx)
|
||||
self.verbose_logging = verbose_logging
|
||||
|
||||
def _identityVerifyingInfoCallback(self, connection, where, ret):
|
||||
|
|
|
|||
|
|
@ -141,10 +141,12 @@ class ExecutionEngine:
|
|||
|
||||
def _needs_backout(self, spider):
|
||||
slot = self.slot
|
||||
return not self.running \
|
||||
or slot.closing \
|
||||
or self.downloader.needs_backout() \
|
||||
return (
|
||||
not self.running
|
||||
or slot.closing
|
||||
or self.downloader.needs_backout()
|
||||
or self.scraper.slot.needs_backout()
|
||||
)
|
||||
|
||||
def _next_request_from_scheduler(self, spider):
|
||||
slot = self.slot
|
||||
|
|
|
|||
|
|
@ -148,7 +148,7 @@ class Scraper:
|
|||
def call_spider(self, result, request, spider):
|
||||
result.request = request
|
||||
dfd = defer_result(result)
|
||||
callback = request.callback or spider.parse
|
||||
callback = request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
dfd.addCallbacks(callback=callback,
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
|
||||
|
||||
def _add_middleware(self, mw):
|
||||
super(SpiderMiddlewareManager, self)._add_middleware(mw)
|
||||
super()._add_middleware(mw)
|
||||
if hasattr(mw, 'process_spider_input'):
|
||||
self.methods['process_spider_input'].append(mw.process_spider_input)
|
||||
if hasattr(mw, 'process_start_requests'):
|
||||
|
|
|
|||
|
|
@ -277,7 +277,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
"""
|
||||
|
||||
def __init__(self, settings=None, install_root_handler=True):
|
||||
super(CrawlerProcess, self).__init__(settings)
|
||||
super().__init__(settings)
|
||||
install_shutdown_handlers(self._signal_shutdown)
|
||||
configure_logging(self.settings, install_root_handler)
|
||||
log_scrapy_info(self.settings)
|
||||
|
|
@ -307,7 +307,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
||||
:param boolean stop_after_crawl: stop or not the reactor when all
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
|
|
|
|||
|
|
@ -33,10 +33,8 @@ class BaseRedirectMiddleware:
|
|||
if ttl and redirects <= self.max_redirect_times:
|
||||
redirected.meta['redirect_times'] = redirects
|
||||
redirected.meta['redirect_ttl'] = ttl - 1
|
||||
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + \
|
||||
[request.url]
|
||||
redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + \
|
||||
[reason]
|
||||
redirected.meta['redirect_urls'] = request.meta.get('redirect_urls', []) + [request.url]
|
||||
redirected.meta['redirect_reasons'] = request.meta.get('redirect_reasons', []) + [reason]
|
||||
redirected.dont_filter = request.dont_filter
|
||||
redirected.priority = request.priority + self.priority_adjust
|
||||
logger.debug("Redirecting (%(reason)s) to %(redirected)s from %(request)s",
|
||||
|
|
@ -94,13 +92,16 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
|
|||
enabled_setting = 'METAREFRESH_ENABLED'
|
||||
|
||||
def __init__(self, settings):
|
||||
super(MetaRefreshMiddleware, self).__init__(settings)
|
||||
super().__init__(settings)
|
||||
self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS')
|
||||
self._maxdelay = settings.getint('METAREFRESH_MAXDELAY')
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
if request.meta.get('dont_redirect', False) or request.method == 'HEAD' or \
|
||||
not isinstance(response, HtmlResponse):
|
||||
if (
|
||||
request.meta.get('dont_redirect', False)
|
||||
or request.method == 'HEAD'
|
||||
or not isinstance(response, HtmlResponse)
|
||||
):
|
||||
return response
|
||||
|
||||
interval, url = get_meta_refresh(response,
|
||||
|
|
|
|||
|
|
@ -60,8 +60,10 @@ class RetryMiddleware:
|
|||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
|
||||
and not request.meta.get('dont_retry', False):
|
||||
if (
|
||||
isinstance(exception, self.EXCEPTIONS_TO_RETRY)
|
||||
and not request.meta.get('dont_retry', False)
|
||||
):
|
||||
return self._retry(request, exception, spider)
|
||||
|
||||
def _retry(self, request, reason, spider):
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ class CloseSpider(Exception):
|
|||
"""Raise this from callbacks to request the spider to be closed"""
|
||||
|
||||
def __init__(self, reason='cancelled'):
|
||||
super(CloseSpider, self).__init__()
|
||||
super().__init__()
|
||||
self.reason = reason
|
||||
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ class UsageError(Exception):
|
|||
|
||||
def __init__(self, *a, **kw):
|
||||
self.print_help = kw.pop('print_help', True)
|
||||
super(UsageError, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
|
||||
|
||||
class ScrapyDeprecationWarning(Warning):
|
||||
|
|
|
|||
|
|
@ -243,12 +243,8 @@ class CsvItemExporter(BaseItemExporter):
|
|||
def _write_headers_and_set_fields_to_export(self, item):
|
||||
if self.include_headers_line:
|
||||
if not self.fields_to_export:
|
||||
if isinstance(item, dict):
|
||||
# for dicts try using fields of the first item
|
||||
self.fields_to_export = list(item.keys())
|
||||
else:
|
||||
# use fields declared in Item
|
||||
self.fields_to_export = list(item.fields.keys())
|
||||
# use declared field names, or keys if the item is a dict
|
||||
self.fields_to_export = ItemAdapter(item).field_names()
|
||||
row = list(self._build_row(self.fields_to_export))
|
||||
self.csv_writer.writerow(row)
|
||||
|
||||
|
|
@ -305,7 +301,7 @@ class PythonItemExporter(BaseItemExporter):
|
|||
|
||||
def _configure(self, options, dont_fail=False):
|
||||
self.binary = options.pop('binary', True)
|
||||
super(PythonItemExporter, self)._configure(options, dont_fail)
|
||||
super()._configure(options, dont_fail)
|
||||
if self.binary:
|
||||
warnings.warn(
|
||||
"PythonItemExporter will drop support for binary export in the future",
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ See documentation in docs/topics/feed-exports.rst
|
|||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
from datetime import datetime
|
||||
|
|
@ -153,6 +154,32 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
key.close()
|
||||
|
||||
|
||||
class GCSFeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri, project_id, acl):
|
||||
self.project_id = project_id
|
||||
self.acl = acl
|
||||
u = urlparse(uri)
|
||||
self.bucket_name = u.hostname
|
||||
self.blob_name = u.path[1:] # remove first "/"
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return cls(
|
||||
uri,
|
||||
crawler.settings['GCS_PROJECT_ID'],
|
||||
crawler.settings['FEED_STORAGE_GCS_ACL'] or None
|
||||
)
|
||||
|
||||
def _store_in_thread(self, file):
|
||||
file.seek(0)
|
||||
from google.cloud.storage import Client
|
||||
client = Client(project=self.project_id)
|
||||
bucket = client.get_bucket(self.bucket_name)
|
||||
blob = bucket.blob(self.blob_name)
|
||||
blob.upload_from_file(file, predefined_acl=self.acl)
|
||||
|
||||
|
||||
class FTPFeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri, use_active_mode=False):
|
||||
|
|
@ -180,14 +207,16 @@ class FTPFeedStorage(BlockingFeedStorage):
|
|||
|
||||
|
||||
class _FeedSlot:
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty):
|
||||
def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
|
||||
self.file = file
|
||||
self.exporter = exporter
|
||||
self.storage = storage
|
||||
# feed params
|
||||
self.uri = uri
|
||||
self.batch_id = batch_id
|
||||
self.format = format
|
||||
self.store_empty = store_empty
|
||||
self.uri_template = uri_template
|
||||
self.uri = uri
|
||||
# flags
|
||||
self.itemcount = 0
|
||||
self._exporting = False
|
||||
|
|
@ -244,63 +273,112 @@ class FeedExporter:
|
|||
for uri, feed in self.feeds.items():
|
||||
if not self._storage_supported(uri):
|
||||
raise NotConfigured
|
||||
if not self._settings_are_valid():
|
||||
raise NotConfigured
|
||||
if not self._exporter_supported(feed['format']):
|
||||
raise NotConfigured
|
||||
|
||||
def open_spider(self, spider):
|
||||
for uri, feed in self.feeds.items():
|
||||
uri = uri % self._get_uri_params(spider, feed['uri_params'])
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed['format'],
|
||||
fields_to_export=feed['fields'],
|
||||
encoding=feed['encoding'],
|
||||
indent=feed['indent'],
|
||||
)
|
||||
slot = _FeedSlot(file, exporter, storage, uri, feed['format'], feed['store_empty'])
|
||||
self.slots.append(slot)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
uri_params = self._get_uri_params(spider, feed['uri_params'])
|
||||
self.slots.append(self._start_new_batch(
|
||||
batch_id=1,
|
||||
uri=uri % uri_params,
|
||||
feed=feed,
|
||||
spider=spider,
|
||||
uri_template=uri,
|
||||
))
|
||||
|
||||
def close_spider(self, spider):
|
||||
deferred_list = []
|
||||
for slot in self.slots:
|
||||
if not slot.itemcount and not slot.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
deferred_list.append(d)
|
||||
continue
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
|
||||
# Use `largs=log_args` to copy log_args into function's scope
|
||||
# instead of using `log_args` from the outer scope
|
||||
d.addCallback(
|
||||
lambda _, largs=log_args: logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
)
|
||||
)
|
||||
d.addErrback(
|
||||
lambda f, largs=log_args: logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
)
|
||||
d = self._close_slot(slot, spider)
|
||||
deferred_list.append(d)
|
||||
return defer.DeferredList(deferred_list) if deferred_list else None
|
||||
|
||||
def _close_slot(self, slot, spider):
|
||||
if not slot.itemcount and not slot.store_empty:
|
||||
# We need to call slot.storage.store nonetheless to get the file
|
||||
# properly closed.
|
||||
return defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
slot.finish_exporting()
|
||||
logfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"
|
||||
log_args = {'format': slot.format,
|
||||
'itemcount': slot.itemcount,
|
||||
'uri': slot.uri}
|
||||
d = defer.maybeDeferred(slot.storage.store, slot.file)
|
||||
|
||||
# Use `largs=log_args` to copy log_args into function's scope
|
||||
# instead of using `log_args` from the outer scope
|
||||
d.addCallback(
|
||||
lambda _, largs=log_args: logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
)
|
||||
)
|
||||
d.addErrback(
|
||||
lambda f, largs=log_args: logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
)
|
||||
return d
|
||||
|
||||
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
|
||||
"""
|
||||
Redirect the output data stream to a new file.
|
||||
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
|
||||
:param batch_id: sequence number of current batch
|
||||
:param uri: uri of the new batch to start
|
||||
:param feed: dict with parameters of feed
|
||||
:param spider: user spider
|
||||
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
|
||||
"""
|
||||
storage = self._get_storage(uri)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed['format'],
|
||||
fields_to_export=feed['fields'],
|
||||
encoding=feed['encoding'],
|
||||
indent=feed['indent'],
|
||||
)
|
||||
slot = _FeedSlot(
|
||||
file=file,
|
||||
exporter=exporter,
|
||||
storage=storage,
|
||||
uri=uri,
|
||||
format=feed['format'],
|
||||
store_empty=feed['store_empty'],
|
||||
batch_id=batch_id,
|
||||
uri_template=uri_template,
|
||||
)
|
||||
if slot.store_empty:
|
||||
slot.start_exporting()
|
||||
return slot
|
||||
|
||||
def item_scraped(self, item, spider):
|
||||
slots = []
|
||||
for slot in self.slots:
|
||||
slot.start_exporting()
|
||||
slot.exporter.export_item(item)
|
||||
slot.itemcount += 1
|
||||
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
|
||||
if (
|
||||
self.feeds[slot.uri_template]['batch_item_count']
|
||||
and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
|
||||
):
|
||||
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
|
||||
self._close_slot(slot, spider)
|
||||
slots.append(self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=slot.uri_template % uri_params,
|
||||
feed=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
uri_template=slot.uri_template,
|
||||
))
|
||||
else:
|
||||
slots.append(slot)
|
||||
self.slots = slots
|
||||
|
||||
def _load_components(self, setting_prefix):
|
||||
conf = without_none_values(self.settings.getwithbase(setting_prefix))
|
||||
|
|
@ -317,6 +395,22 @@ class FeedExporter:
|
|||
return True
|
||||
logger.error("Unknown feed format: %(format)s", {'format': format})
|
||||
|
||||
def _settings_are_valid(self):
|
||||
"""
|
||||
If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
|
||||
%(batch_time)s or %(batch_id)d to distinguish different files of partial output
|
||||
"""
|
||||
for uri_template, values in self.feeds.items():
|
||||
if values['batch_item_count'] and not re.search(r'%\(batch_time\)s|%\(batch_id\)', uri_template):
|
||||
logger.error(
|
||||
'%(batch_time)s or %(batch_id)d must be in the feed URI ({}) if FEED_EXPORT_BATCH_ITEM_COUNT '
|
||||
'setting or FEEDS.batch_item_count is specified and greater than 0. For more info see: '
|
||||
'https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-batch-item-count'
|
||||
''.format(uri_template)
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def _storage_supported(self, uri):
|
||||
scheme = urlparse(uri).scheme
|
||||
if scheme in self.storages:
|
||||
|
|
@ -342,12 +436,14 @@ class FeedExporter:
|
|||
def _get_storage(self, uri):
|
||||
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
|
||||
|
||||
def _get_uri_params(self, spider, uri_params):
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
params = {}
|
||||
for k in dir(spider):
|
||||
params[k] = getattr(spider, k)
|
||||
ts = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['time'] = ts
|
||||
utc_now = datetime.utcnow()
|
||||
params['time'] = utc_now.replace(microsecond=0).isoformat().replace(':', '-')
|
||||
params['batch_time'] = utc_now.isoformat().replace(':', '-')
|
||||
params['batch_id'] = slot.batch_id + 1 if slot is not None else 1
|
||||
uripar_function = load_object(uri_params) if uri_params else lambda x, y: None
|
||||
uripar_function(params, spider)
|
||||
return params
|
||||
|
|
|
|||
|
|
@ -81,8 +81,10 @@ class MemoryUsage:
|
|||
logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
subj = "%s terminated: memory usage exceeded %dM at %s" % \
|
||||
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
subj = (
|
||||
"%s terminated: memory usage exceeded %dM at %s"
|
||||
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
)
|
||||
self._send_report(self.notify_mails, subj)
|
||||
self.crawler.stats.set_value('memusage/limit_notified', 1)
|
||||
|
||||
|
|
@ -102,8 +104,10 @@ class MemoryUsage:
|
|||
logger.warning("Memory usage reached %(memusage)dM",
|
||||
{'memusage': mem}, extra={'crawler': self.crawler})
|
||||
if self.notify_mails:
|
||||
subj = "%s warning: memory usage reached %dM at %s" % \
|
||||
(self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
subj = (
|
||||
"%s warning: memory usage reached %dM at %s"
|
||||
% (self.crawler.settings['BOT_NAME'], mem, socket.gethostname())
|
||||
)
|
||||
self._send_report(self.notify_mails, subj)
|
||||
self.crawler.stats.set_value('memusage/warning_notified', 1)
|
||||
self.warned = True
|
||||
|
|
|
|||
|
|
@ -186,9 +186,6 @@ class WrappedResponse:
|
|||
def info(self):
|
||||
return self
|
||||
|
||||
# python3 cookiejars calls get_all
|
||||
def get_all(self, name, default=None):
|
||||
return [to_unicode(v, errors='replace')
|
||||
for v in self.response.headers.getlist(name)]
|
||||
# python2 cookiejars calls getheaders
|
||||
getheaders = get_all
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ class Headers(CaselessDict):
|
|||
|
||||
def __init__(self, seq=None, encoding='utf-8'):
|
||||
self.encoding = encoding
|
||||
super(Headers, self).__init__(seq)
|
||||
super().__init__(seq)
|
||||
|
||||
def normkey(self, key):
|
||||
"""Normalize key to bytes"""
|
||||
|
|
@ -37,19 +37,19 @@ class Headers(CaselessDict):
|
|||
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return super(Headers, self).__getitem__(key)[-1]
|
||||
return super().__getitem__(key)[-1]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def get(self, key, def_val=None):
|
||||
try:
|
||||
return super(Headers, self).get(key, def_val)[-1]
|
||||
return super().get(key, def_val)[-1]
|
||||
except IndexError:
|
||||
return None
|
||||
|
||||
def getlist(self, key, def_val=None):
|
||||
try:
|
||||
return super(Headers, self).__getitem__(key)
|
||||
return super().__getitem__(key)
|
||||
except KeyError:
|
||||
if def_val is not None:
|
||||
return self.normvalue(def_val)
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ class FormRequest(Request):
|
|||
if formdata and kwargs.get('method') is None:
|
||||
kwargs['method'] = 'POST'
|
||||
|
||||
super(FormRequest, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
if formdata:
|
||||
items = formdata.items() if isinstance(formdata, dict) else formdata
|
||||
|
|
@ -133,7 +133,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
|
|||
' not(re:test(., "^(?:checkbox|radio)$", "i")))]]',
|
||||
namespaces={
|
||||
"re": "http://exslt.org/regular-expressions"})
|
||||
values = [(k, u'' if v is None else v)
|
||||
values = [(k, '' if v is None else v)
|
||||
for k, v in (_value(e) for e in inputs)
|
||||
if k and k not in formdata_keys]
|
||||
|
||||
|
|
@ -168,7 +168,7 @@ def _select_value(ele, n, v):
|
|||
# This is a workround to bug in lxml fixed 2.3.1
|
||||
# fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139
|
||||
selected_options = ele.xpath('.//option[@selected]')
|
||||
v = [(o.get('value') or o.text or u'').strip() for o in selected_options]
|
||||
v = [(o.get('value') or o.text or '').strip() for o in selected_options]
|
||||
return n, v
|
||||
|
||||
|
||||
|
|
@ -205,8 +205,7 @@ def _get_clickable(clickdata, form):
|
|||
|
||||
# We didn't find it, so now we build an XPath expression out of the other
|
||||
# arguments, because they can be used as such
|
||||
xpath = u'.//*' + \
|
||||
u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
|
||||
xpath = './/*' + ''.join('[@%s="%s"]' % c for c in clickdata.items())
|
||||
el = form.xpath(xpath)
|
||||
if len(el) == 1:
|
||||
return (el[0].get('name'), el[0].get('value') or '')
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ class JsonRequest(Request):
|
|||
if 'method' not in kwargs:
|
||||
kwargs['method'] = 'POST'
|
||||
|
||||
super(JsonRequest, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.headers.setdefault('Content-Type', 'application/json')
|
||||
self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01')
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ class JsonRequest(Request):
|
|||
elif not body_passed and data_passed:
|
||||
kwargs['body'] = self._dumps(data)
|
||||
|
||||
return super(JsonRequest, self).replace(*args, **kwargs)
|
||||
return super().replace(*args, **kwargs)
|
||||
|
||||
def _dumps(self, data):
|
||||
"""Convert to JSON """
|
||||
|
|
|
|||
|
|
@ -31,5 +31,5 @@ class XmlRpcRequest(Request):
|
|||
if encoding is not None:
|
||||
kwargs['encoding'] = encoding
|
||||
|
||||
super(XmlRpcRequest, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.headers.setdefault('Content-Type', 'text/xml')
|
||||
|
|
|
|||
|
|
@ -35,13 +35,13 @@ class TextResponse(Response):
|
|||
self._cached_benc = None
|
||||
self._cached_ubody = None
|
||||
self._cached_selector = None
|
||||
super(TextResponse, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, str):
|
||||
self._url = to_unicode(url, self.encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_url(url)
|
||||
super()._set_url(url)
|
||||
|
||||
def _set_body(self, body):
|
||||
self._body = b'' # used by encoding detection
|
||||
|
|
@ -51,7 +51,7 @@ class TextResponse(Response):
|
|||
type(self).__name__)
|
||||
self._body = body.encode(self._encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_body(body)
|
||||
super()._set_body(body)
|
||||
|
||||
def replace(self, *args, **kwargs):
|
||||
kwargs.setdefault('encoding', self.encoding)
|
||||
|
|
@ -62,8 +62,11 @@ class TextResponse(Response):
|
|||
return self._declared_encoding() or self._body_inferred_encoding()
|
||||
|
||||
def _declared_encoding(self):
|
||||
return self._encoding or self._headers_encoding() \
|
||||
return (
|
||||
self._encoding
|
||||
or self._headers_encoding()
|
||||
or self._body_declared_encoding()
|
||||
)
|
||||
|
||||
def body_as_unicode(self):
|
||||
"""Return body as unicode"""
|
||||
|
|
@ -163,7 +166,7 @@ class TextResponse(Response):
|
|||
elif isinstance(url, parsel.SelectorList):
|
||||
raise ValueError("SelectorList is not supported")
|
||||
encoding = self.encoding if encoding is None else encoding
|
||||
return super(TextResponse, self).follow(
|
||||
return super().follow(
|
||||
url=url,
|
||||
callback=callback,
|
||||
method=method,
|
||||
|
|
@ -223,7 +226,7 @@ class TextResponse(Response):
|
|||
for sel in selectors:
|
||||
with suppress(_InvalidSelector):
|
||||
urls.append(_url_from_selector(sel))
|
||||
return super(TextResponse, self).follow_all(
|
||||
return super().follow_all(
|
||||
urls=urls,
|
||||
callback=callback,
|
||||
method=method,
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class BaseItem(_BaseItem, metaclass=_BaseItemMeta):
|
|||
if issubclass(cls, BaseItem) and not issubclass(cls, (Item, DictItem)):
|
||||
warn('scrapy.item.BaseItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(BaseItem, cls).__new__(cls, *args, **kwargs)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
|
||||
class Field(dict):
|
||||
|
|
@ -55,7 +55,7 @@ class ItemMeta(_BaseItemMeta):
|
|||
def __new__(mcs, class_name, bases, attrs):
|
||||
classcell = attrs.pop('__classcell__', None)
|
||||
new_bases = tuple(base._class for base in bases if hasattr(base, '_class'))
|
||||
_class = super(ItemMeta, mcs).__new__(mcs, 'x_' + class_name, new_bases, attrs)
|
||||
_class = super().__new__(mcs, 'x_' + class_name, new_bases, attrs)
|
||||
|
||||
fields = getattr(_class, 'fields', {})
|
||||
new_attrs = {}
|
||||
|
|
@ -70,7 +70,7 @@ class ItemMeta(_BaseItemMeta):
|
|||
new_attrs['_class'] = _class
|
||||
if classcell is not None:
|
||||
new_attrs['__classcell__'] = classcell
|
||||
return super(ItemMeta, mcs).__new__(mcs, class_name, bases, new_attrs)
|
||||
return super().__new__(mcs, class_name, bases, new_attrs)
|
||||
|
||||
|
||||
class DictItem(MutableMapping, BaseItem):
|
||||
|
|
@ -81,7 +81,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
if issubclass(cls, DictItem) and not issubclass(cls, Item):
|
||||
warn('scrapy.item.DictItem is deprecated, please use scrapy.item.Item instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(DictItem, cls).__new__(cls, *args, **kwargs)
|
||||
return super().__new__(cls, *args, **kwargs)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
self._values = {}
|
||||
|
|
@ -109,7 +109,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
def __setattr__(self, name, value):
|
||||
if not name.startswith('_'):
|
||||
raise AttributeError("Use item[%r] = %r to set field value" % (name, value))
|
||||
super(DictItem, self).__setattr__(name, value)
|
||||
super().__setattr__(name, value)
|
||||
|
||||
def __len__(self):
|
||||
return len(self._values)
|
||||
|
|
|
|||
|
|
@ -21,12 +21,18 @@ class Link:
|
|||
self.nofollow = nofollow
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.url == other.url and self.text == other.text and \
|
||||
self.fragment == other.fragment and self.nofollow == other.nofollow
|
||||
return (
|
||||
self.url == other.url
|
||||
and self.text == other.text
|
||||
and self.fragment == other.fragment
|
||||
and self.nofollow == other.nofollow
|
||||
)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.url) ^ hash(self.text) ^ hash(self.fragment) ^ hash(self.nofollow)
|
||||
|
||||
def __repr__(self):
|
||||
return 'Link(url=%r, text=%r, fragment=%r, nofollow=%r)' % \
|
||||
(self.url, self.text, self.fragment, self.nofollow)
|
||||
return (
|
||||
'Link(url=%r, text=%r, fragment=%r, nofollow=%r)'
|
||||
% (self.url, self.text, self.fragment, self.nofollow)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -65,7 +65,7 @@ class FilteringLinkExtractor:
|
|||
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
|
||||
'please use scrapy.linkextractors.LinkExtractor instead',
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
return super(FilteringLinkExtractor, cls).__new__(cls)
|
||||
return super().__new__(cls)
|
||||
|
||||
def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains,
|
||||
restrict_xpaths, canonicalize, deny_extensions, restrict_css, restrict_text):
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
"""
|
||||
HTMLParser-based link extractor
|
||||
"""
|
||||
import warnings
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
class HtmlParserLinkExtractor(HTMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", process=None, unique=False,
|
||||
strip=True):
|
||||
HTMLParser.__init__(self)
|
||||
|
||||
warnings.warn(
|
||||
"HtmlParserLinkExtractor is deprecated and will be removed in "
|
||||
"future releases. Please use scrapy.linkextractors.LinkExtractor",
|
||||
ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_attr = process if callable(process) else lambda v: v
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding):
|
||||
self.reset()
|
||||
self.feed(response_text)
|
||||
self.close()
|
||||
|
||||
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
|
||||
|
||||
ret = []
|
||||
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in links:
|
||||
if isinstance(link.url, str):
|
||||
link.url = link.url.encode(response_encoding)
|
||||
try:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
except ValueError:
|
||||
continue
|
||||
link.url = safe_url_string(link.url, response_encoding)
|
||||
link.text = link.text.decode(response_encoding)
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
return self._extract_links(response.body, response.url, response.encoding)
|
||||
|
||||
def reset(self):
|
||||
HTMLParser.reset(self)
|
||||
|
||||
self.base_url = None
|
||||
self.current_link = None
|
||||
self.links = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == 'base':
|
||||
self.base_url = dict(attrs).get('href')
|
||||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if self.strip:
|
||||
value = strip_html5_whitespace(value)
|
||||
url = self.process_attr(value)
|
||||
link = Link(url=url)
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.scan_tag(tag):
|
||||
self.current_link = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.current_link:
|
||||
self.current_link.text = self.current_link.text + data
|
||||
|
||||
def matches(self, url):
|
||||
"""This extractor matches with any url, since
|
||||
it doesn't contain any patterns"""
|
||||
return True
|
||||
|
|
@ -76,7 +76,7 @@ class LxmlParserLinkExtractor:
|
|||
url = safe_url_string(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
link = Link(url, _collect_string_content(el) or '',
|
||||
nofollow=rel_has_nofollow(el.get('rel')))
|
||||
links.append(link)
|
||||
return self._deduplicate_if_needed(links)
|
||||
|
|
@ -126,7 +126,7 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
strip=strip,
|
||||
canonicalized=canonicalize
|
||||
)
|
||||
super(LxmlLinkExtractor, self).__init__(
|
||||
super().__init__(
|
||||
link_extractor=lx,
|
||||
allow=allow,
|
||||
deny=deny,
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
import re
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors.sgml import SgmlLinkExtractor
|
||||
|
||||
|
||||
linkre = re.compile(
|
||||
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
|
||||
re.DOTALL | re.IGNORECASE)
|
||||
|
||||
|
||||
def clean_link(link_text):
|
||||
"""Remove leading and trailing whitespace and punctuation"""
|
||||
return link_text.strip("\t\r\n '\"\x0c")
|
||||
|
||||
|
||||
class RegexLinkExtractor(SgmlLinkExtractor):
|
||||
"""High performant link extractor"""
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
|
||||
def clean_text(text):
|
||||
return replace_escape_chars(remove_tags(text.decode(response_encoding))).strip()
|
||||
|
||||
def clean_url(url):
|
||||
clean_url = ''
|
||||
try:
|
||||
clean_url = urljoin(base_url, replace_entities(clean_link(url.decode(response_encoding))))
|
||||
except ValueError:
|
||||
pass
|
||||
return clean_url
|
||||
|
||||
if base_url is None:
|
||||
base_url = get_base_url(response_text, response_url, response_encoding)
|
||||
|
||||
links_text = linkre.findall(response_text)
|
||||
return [Link(clean_url(url).encode(response_encoding),
|
||||
clean_text(text))
|
||||
for url, _, text in links_text]
|
||||
|
|
@ -1,151 +0,0 @@
|
|||
"""
|
||||
SGMLParser-based Link extractors
|
||||
"""
|
||||
import warnings
|
||||
from urllib.parse import urljoin
|
||||
from sgmllib import SGMLParser
|
||||
|
||||
from w3lib.url import safe_url_string, canonicalize_url
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list, to_unicode
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
class BaseSgmlLinkExtractor(SGMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", unique=False, process_value=None,
|
||||
strip=True, canonicalized=False):
|
||||
warnings.warn(
|
||||
"BaseSgmlLinkExtractor is deprecated and will be removed in future releases. "
|
||||
"Please use scrapy.linkextractors.LinkExtractor",
|
||||
ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
SGMLParser.__init__(self)
|
||||
self.scan_tag = tag if callable(tag) else lambda t: t == tag
|
||||
self.scan_attr = attr if callable(attr) else lambda a: a == attr
|
||||
self.process_value = (lambda v: v) if process_value is None else process_value
|
||||
self.current_link = None
|
||||
self.unique = unique
|
||||
self.strip = strip
|
||||
if canonicalized:
|
||||
self.link_key = lambda link: link.url
|
||||
else:
|
||||
self.link_key = lambda link: canonicalize_url(link.url,
|
||||
keep_fragments=True)
|
||||
|
||||
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
|
||||
""" Do the real extraction work """
|
||||
self.reset()
|
||||
self.feed(response_text)
|
||||
self.close()
|
||||
|
||||
ret = []
|
||||
if base_url is None:
|
||||
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
|
||||
for link in self.links:
|
||||
if isinstance(link.url, str):
|
||||
link.url = link.url.encode(response_encoding)
|
||||
try:
|
||||
link.url = urljoin(base_url, link.url)
|
||||
except ValueError:
|
||||
continue
|
||||
link.url = safe_url_string(link.url, response_encoding)
|
||||
link.text = to_unicode(link.text, response_encoding, errors='replace').strip()
|
||||
ret.append(link)
|
||||
|
||||
return ret
|
||||
|
||||
def _process_links(self, links):
|
||||
""" Normalize and filter extracted links
|
||||
|
||||
The subclass should override it if necessary
|
||||
"""
|
||||
return unique_list(links, key=self.link_key) if self.unique else links
|
||||
|
||||
def extract_links(self, response):
|
||||
# wrapper needed to allow to work directly with text
|
||||
links = self._extract_links(response.body, response.url, response.encoding)
|
||||
links = self._process_links(links)
|
||||
return links
|
||||
|
||||
def reset(self):
|
||||
SGMLParser.reset(self)
|
||||
self.links = []
|
||||
self.base_url = None
|
||||
self.current_link = None
|
||||
|
||||
def unknown_starttag(self, tag, attrs):
|
||||
if tag == 'base':
|
||||
self.base_url = dict(attrs).get('href')
|
||||
if self.scan_tag(tag):
|
||||
for attr, value in attrs:
|
||||
if self.scan_attr(attr):
|
||||
if self.strip and value is not None:
|
||||
value = strip_html5_whitespace(value)
|
||||
url = self.process_value(value)
|
||||
if url is not None:
|
||||
link = Link(url=url, nofollow=rel_has_nofollow(dict(attrs).get('rel')))
|
||||
self.links.append(link)
|
||||
self.current_link = link
|
||||
|
||||
def unknown_endtag(self, tag):
|
||||
if self.scan_tag(tag):
|
||||
self.current_link = None
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.current_link:
|
||||
self.current_link.text = self.current_link.text + data
|
||||
|
||||
def matches(self, url):
|
||||
"""This extractor matches with any url, since
|
||||
it doesn't contain any patterns"""
|
||||
return True
|
||||
|
||||
|
||||
class SgmlLinkExtractor(FilteringLinkExtractor):
|
||||
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True,
|
||||
process_value=None, deny_extensions=None, restrict_css=(),
|
||||
strip=True, restrict_text=()):
|
||||
warnings.warn(
|
||||
"SgmlLinkExtractor is deprecated and will be removed in future releases. "
|
||||
"Please use scrapy.linkextractors.LinkExtractor",
|
||||
ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func,
|
||||
unique=unique, process_value=process_value, strip=strip,
|
||||
canonicalized=canonicalize)
|
||||
|
||||
super(SgmlLinkExtractor, self).__init__(lx, allow=allow, deny=deny,
|
||||
allow_domains=allow_domains, deny_domains=deny_domains,
|
||||
restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
|
||||
canonicalize=canonicalize, deny_extensions=deny_extensions,
|
||||
restrict_text=restrict_text)
|
||||
|
||||
def extract_links(self, response):
|
||||
base_url = None
|
||||
if self.restrict_xpaths:
|
||||
base_url = get_base_url(response)
|
||||
body = u''.join(f
|
||||
for x in self.restrict_xpaths
|
||||
for f in response.xpath(x).getall()
|
||||
).encode(response.encoding, errors='xmlcharrefreplace')
|
||||
else:
|
||||
body = response.body
|
||||
|
||||
links = self._extract_links(body, response.url, response.encoding, base_url)
|
||||
links = self._process_links(links)
|
||||
return links
|
||||
|
|
@ -3,217 +3,86 @@ Item Loader
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from collections import defaultdict
|
||||
from contextlib import suppress
|
||||
|
||||
from itemadapter import ItemAdapter
|
||||
import itemloaders
|
||||
|
||||
from scrapy.item import Item
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.loader.processors import Identity
|
||||
from scrapy.selector import Selector
|
||||
from scrapy.utils.misc import arg_to_iter, extract_regex
|
||||
from scrapy.utils.python import flatten
|
||||
|
||||
|
||||
def unbound_method(method):
|
||||
class ItemLoader(itemloaders.ItemLoader):
|
||||
"""
|
||||
Allow to use single-argument functions as input or output processors
|
||||
(no need to define an unused first 'self' argument)
|
||||
A user-friendly abstraction to populate an :ref:`item <topics-items>` with data
|
||||
by applying :ref:`field processors <topics-loaders-processors>` to scraped data.
|
||||
When instantiated with a ``selector`` or a ``response`` it supports
|
||||
data extraction from web pages using :ref:`selectors <topics-selectors>`.
|
||||
|
||||
:param item: The item instance to populate using subsequent calls to
|
||||
:meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
|
||||
or :meth:`~ItemLoader.add_value`.
|
||||
:type item: scrapy.item.Item
|
||||
|
||||
:param selector: The selector to extract data from, when using the
|
||||
:meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or
|
||||
:meth:`replace_css` method.
|
||||
:type selector: :class:`~scrapy.selector.Selector` object
|
||||
|
||||
:param response: The response used to construct the selector using the
|
||||
:attr:`default_selector_class`, unless the selector argument is given,
|
||||
in which case this argument is ignored.
|
||||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
If no item is given, one is instantiated automatically using the class in
|
||||
:attr:`default_item_class`.
|
||||
|
||||
The item, selector, response and remaining keyword arguments are
|
||||
assigned to the Loader context (accessible through the :attr:`context` attribute).
|
||||
|
||||
.. attribute:: item
|
||||
|
||||
The item object being parsed by this Item Loader.
|
||||
This is mostly used as a property so, when attempting to override this
|
||||
value, you may want to check out :attr:`default_item_class` first.
|
||||
|
||||
.. attribute:: context
|
||||
|
||||
The currently active :ref:`Context <loaders-context>` of this Item Loader.
|
||||
|
||||
.. attribute:: default_item_class
|
||||
|
||||
An :ref:`item <topics-items>` class (or factory), used to instantiate
|
||||
items when not given in the ``__init__`` method.
|
||||
|
||||
.. attribute:: default_input_processor
|
||||
|
||||
The default input processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_output_processor
|
||||
|
||||
The default output processor to use for those fields which don't specify
|
||||
one.
|
||||
|
||||
.. attribute:: default_selector_class
|
||||
|
||||
The class used to construct the :attr:`selector` of this
|
||||
:class:`ItemLoader`, if only a response is given in the ``__init__`` method.
|
||||
If a selector is given in the ``__init__`` method this attribute is ignored.
|
||||
This attribute is sometimes overridden in subclasses.
|
||||
|
||||
.. attribute:: selector
|
||||
|
||||
The :class:`~scrapy.selector.Selector` object to extract data from.
|
||||
It's either the selector given in the ``__init__`` method or one created from
|
||||
the response given in the ``__init__`` method using the
|
||||
:attr:`default_selector_class`. This attribute is meant to be
|
||||
read-only.
|
||||
"""
|
||||
with suppress(AttributeError):
|
||||
if '.' not in method.__qualname__:
|
||||
return method.__func__
|
||||
return method
|
||||
|
||||
|
||||
class ItemLoader:
|
||||
|
||||
default_item_class = Item
|
||||
default_input_processor = Identity()
|
||||
default_output_processor = Identity()
|
||||
default_selector_class = Selector
|
||||
|
||||
def __init__(self, item=None, selector=None, response=None, parent=None, **context):
|
||||
if selector is None and response is not None:
|
||||
selector = self.default_selector_class(response)
|
||||
self.selector = selector
|
||||
context.update(selector=selector, response=response)
|
||||
if item is None:
|
||||
item = self.default_item_class()
|
||||
self.context = context
|
||||
self.parent = parent
|
||||
self._local_item = context['item'] = item
|
||||
self._local_values = defaultdict(list)
|
||||
# values from initial item
|
||||
for field_name, value in ItemAdapter(item).items():
|
||||
self._values[field_name] += arg_to_iter(value)
|
||||
|
||||
@property
|
||||
def _values(self):
|
||||
if self.parent is not None:
|
||||
return self.parent._values
|
||||
else:
|
||||
return self._local_values
|
||||
|
||||
@property
|
||||
def item(self):
|
||||
if self.parent is not None:
|
||||
return self.parent.item
|
||||
else:
|
||||
return self._local_item
|
||||
|
||||
def nested_xpath(self, xpath, **context):
|
||||
selector = self.selector.xpath(xpath)
|
||||
context.update(selector=selector)
|
||||
subloader = self.__class__(
|
||||
item=self.item, parent=self, **context
|
||||
)
|
||||
return subloader
|
||||
|
||||
def nested_css(self, css, **context):
|
||||
selector = self.selector.css(css)
|
||||
context.update(selector=selector)
|
||||
subloader = self.__class__(
|
||||
item=self.item, parent=self, **context
|
||||
)
|
||||
return subloader
|
||||
|
||||
def add_value(self, field_name, value, *processors, **kw):
|
||||
value = self.get_value(value, *processors, **kw)
|
||||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in value.items():
|
||||
self._add_value(k, v)
|
||||
else:
|
||||
self._add_value(field_name, value)
|
||||
|
||||
def replace_value(self, field_name, value, *processors, **kw):
|
||||
value = self.get_value(value, *processors, **kw)
|
||||
if value is None:
|
||||
return
|
||||
if not field_name:
|
||||
for k, v in value.items():
|
||||
self._replace_value(k, v)
|
||||
else:
|
||||
self._replace_value(field_name, value)
|
||||
|
||||
def _add_value(self, field_name, value):
|
||||
value = arg_to_iter(value)
|
||||
processed_value = self._process_input_value(field_name, value)
|
||||
if processed_value:
|
||||
self._values[field_name] += arg_to_iter(processed_value)
|
||||
|
||||
def _replace_value(self, field_name, value):
|
||||
self._values.pop(field_name, None)
|
||||
self._add_value(field_name, value)
|
||||
|
||||
def get_value(self, value, *processors, **kw):
|
||||
regex = kw.get('re', None)
|
||||
if regex:
|
||||
value = arg_to_iter(value)
|
||||
value = flatten(extract_regex(regex, x) for x in value)
|
||||
|
||||
for proc in processors:
|
||||
if value is None:
|
||||
break
|
||||
_proc = proc
|
||||
proc = wrap_loader_context(proc, self.context)
|
||||
try:
|
||||
value = proc(value)
|
||||
except Exception as e:
|
||||
raise ValueError("Error with processor %s value=%r error='%s: %s'" %
|
||||
(_proc.__class__.__name__, value,
|
||||
type(e).__name__, str(e)))
|
||||
return value
|
||||
|
||||
def load_item(self):
|
||||
adapter = ItemAdapter(self.item)
|
||||
for field_name in tuple(self._values):
|
||||
value = self.get_output_value(field_name)
|
||||
if value is not None:
|
||||
adapter[field_name] = value
|
||||
return adapter.item
|
||||
|
||||
def get_output_value(self, field_name):
|
||||
proc = self.get_output_processor(field_name)
|
||||
proc = wrap_loader_context(proc, self.context)
|
||||
try:
|
||||
return proc(self._values[field_name])
|
||||
except Exception as e:
|
||||
raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" %
|
||||
(field_name, self._values[field_name], type(e).__name__, str(e)))
|
||||
|
||||
def get_collected_values(self, field_name):
|
||||
return self._values[field_name]
|
||||
|
||||
def get_input_processor(self, field_name):
|
||||
proc = getattr(self, '%s_in' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'input_processor',
|
||||
self.default_input_processor)
|
||||
return unbound_method(proc)
|
||||
|
||||
def get_output_processor(self, field_name):
|
||||
proc = getattr(self, '%s_out' % field_name, None)
|
||||
if not proc:
|
||||
proc = self._get_item_field_attr(field_name, 'output_processor',
|
||||
self.default_output_processor)
|
||||
return unbound_method(proc)
|
||||
|
||||
def _process_input_value(self, field_name, value):
|
||||
proc = self.get_input_processor(field_name)
|
||||
_proc = proc
|
||||
proc = wrap_loader_context(proc, self.context)
|
||||
try:
|
||||
return proc(value)
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
"Error with input processor %s: field=%r value=%r "
|
||||
"error='%s: %s'" % (_proc.__class__.__name__, field_name,
|
||||
value, type(e).__name__, str(e)))
|
||||
|
||||
def _get_item_field_attr(self, field_name, key, default=None):
|
||||
field_meta = ItemAdapter(self.item).get_field_meta(field_name)
|
||||
return field_meta.get(key, default)
|
||||
|
||||
def _check_selector_method(self):
|
||||
if self.selector is None:
|
||||
raise RuntimeError("To use XPath or CSS selectors, "
|
||||
"%s must be instantiated with a selector "
|
||||
"or a response" % self.__class__.__name__)
|
||||
|
||||
def add_xpath(self, field_name, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
self.add_value(field_name, values, *processors, **kw)
|
||||
|
||||
def replace_xpath(self, field_name, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
self.replace_value(field_name, values, *processors, **kw)
|
||||
|
||||
def get_xpath(self, xpath, *processors, **kw):
|
||||
values = self._get_xpathvalues(xpath, **kw)
|
||||
return self.get_value(values, *processors, **kw)
|
||||
|
||||
def _get_xpathvalues(self, xpaths, **kw):
|
||||
self._check_selector_method()
|
||||
xpaths = arg_to_iter(xpaths)
|
||||
return flatten(self.selector.xpath(xpath).getall() for xpath in xpaths)
|
||||
|
||||
def add_css(self, field_name, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
self.add_value(field_name, values, *processors, **kw)
|
||||
|
||||
def replace_css(self, field_name, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
self.replace_value(field_name, values, *processors, **kw)
|
||||
|
||||
def get_css(self, css, *processors, **kw):
|
||||
values = self._get_cssvalues(css, **kw)
|
||||
return self.get_value(values, *processors, **kw)
|
||||
|
||||
def _get_cssvalues(self, csss, **kw):
|
||||
self._check_selector_method()
|
||||
csss = arg_to_iter(csss)
|
||||
return flatten(self.selector.css(css).getall() for css in csss)
|
||||
context.update(response=response)
|
||||
super().__init__(item=item, selector=selector, parent=parent, **context)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,21 @@
|
|||
"""Common functions used in Item Loaders code"""
|
||||
|
||||
from functools import partial
|
||||
from scrapy.utils.python import get_func_args
|
||||
import warnings
|
||||
|
||||
from itemloaders import common
|
||||
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
def wrap_loader_context(function, context):
|
||||
"""Wrap functions that receive loader_context to contain the context
|
||||
"pre-loaded" and expose a interface that receives only one argument
|
||||
"""
|
||||
if 'loader_context' in get_func_args(function):
|
||||
return partial(function, loader_context=context)
|
||||
else:
|
||||
return function
|
||||
warnings.warn(
|
||||
"scrapy.loader.common.wrap_loader_context has moved to a new library."
|
||||
"Please update your reference to itemloaders.common.wrap_loader_context",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
return common.wrap_loader_context(function, context)
|
||||
|
|
|
|||
|
|
@ -3,102 +3,19 @@ This module provides some commonly used processors for Item Loaders.
|
|||
|
||||
See documentation in docs/topics/loaders.rst
|
||||
"""
|
||||
from collections import ChainMap
|
||||
from itemloaders import processors
|
||||
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.loader.common import wrap_loader_context
|
||||
from scrapy.utils.deprecate import create_deprecated_class
|
||||
|
||||
|
||||
class MapCompose:
|
||||
MapCompose = create_deprecated_class('MapCompose', processors.MapCompose)
|
||||
|
||||
def __init__(self, *functions, **default_loader_context):
|
||||
self.functions = functions
|
||||
self.default_loader_context = default_loader_context
|
||||
Compose = create_deprecated_class('Compose', processors.Compose)
|
||||
|
||||
def __call__(self, value, loader_context=None):
|
||||
values = arg_to_iter(value)
|
||||
if loader_context:
|
||||
context = ChainMap(loader_context, self.default_loader_context)
|
||||
else:
|
||||
context = self.default_loader_context
|
||||
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
|
||||
for func in wrapped_funcs:
|
||||
next_values = []
|
||||
for v in values:
|
||||
try:
|
||||
next_values += arg_to_iter(func(v))
|
||||
except Exception as e:
|
||||
raise ValueError("Error in MapCompose with "
|
||||
"%s value=%r error='%s: %s'" %
|
||||
(str(func), value, type(e).__name__,
|
||||
str(e)))
|
||||
values = next_values
|
||||
return values
|
||||
TakeFirst = create_deprecated_class('TakeFirst', processors.TakeFirst)
|
||||
|
||||
Identity = create_deprecated_class('Identity', processors.Identity)
|
||||
|
||||
class Compose:
|
||||
SelectJmes = create_deprecated_class('SelectJmes', processors.SelectJmes)
|
||||
|
||||
def __init__(self, *functions, **default_loader_context):
|
||||
self.functions = functions
|
||||
self.stop_on_none = default_loader_context.get('stop_on_none', True)
|
||||
self.default_loader_context = default_loader_context
|
||||
|
||||
def __call__(self, value, loader_context=None):
|
||||
if loader_context:
|
||||
context = ChainMap(loader_context, self.default_loader_context)
|
||||
else:
|
||||
context = self.default_loader_context
|
||||
wrapped_funcs = [wrap_loader_context(f, context) for f in self.functions]
|
||||
for func in wrapped_funcs:
|
||||
if value is None and self.stop_on_none:
|
||||
break
|
||||
try:
|
||||
value = func(value)
|
||||
except Exception as e:
|
||||
raise ValueError("Error in Compose with "
|
||||
"%s value=%r error='%s: %s'" %
|
||||
(str(func), value, type(e).__name__, str(e)))
|
||||
return value
|
||||
|
||||
|
||||
class TakeFirst:
|
||||
|
||||
def __call__(self, values):
|
||||
for value in values:
|
||||
if value is not None and value != '':
|
||||
return value
|
||||
|
||||
|
||||
class Identity:
|
||||
|
||||
def __call__(self, values):
|
||||
return values
|
||||
|
||||
|
||||
class SelectJmes:
|
||||
"""
|
||||
Query the input string for the jmespath (given at instantiation),
|
||||
and return the answer
|
||||
Requires : jmespath(https://github.com/jmespath/jmespath)
|
||||
Note: SelectJmes accepts only one input element at a time.
|
||||
"""
|
||||
def __init__(self, json_path):
|
||||
self.json_path = json_path
|
||||
import jmespath
|
||||
self.compiled_path = jmespath.compile(self.json_path)
|
||||
|
||||
def __call__(self, value):
|
||||
"""Query value for the jmespath query and return answer
|
||||
:param value: a data structure (dict, list) to extract from
|
||||
:return: Element extracted according to jmespath query
|
||||
"""
|
||||
return self.compiled_path.search(value)
|
||||
|
||||
|
||||
class Join:
|
||||
|
||||
def __init__(self, separator=u' '):
|
||||
self.separator = separator
|
||||
|
||||
def __call__(self, values):
|
||||
return self.separator.join(values)
|
||||
Join = create_deprecated_class('Join', processors.Join)
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ class LogFormatter:
|
|||
def dropped(self, item, exception, response, spider):
|
||||
return {
|
||||
'level': logging.INFO, # lowering the level from logging.WARNING
|
||||
'msg': u"Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
|
||||
'args': {
|
||||
'exception': exception,
|
||||
'item': item,
|
||||
|
|
|
|||
|
|
@ -376,7 +376,7 @@ class FilesPipeline(MediaPipeline):
|
|||
resolve('FILES_RESULT_FIELD'), self.FILES_RESULT_FIELD
|
||||
)
|
||||
|
||||
super(FilesPipeline, self).__init__(download_func=download_func, settings=settings)
|
||||
super().__init__(download_func=download_func, settings=settings)
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings):
|
||||
|
|
@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline):
|
|||
store_cls = self.STORE_SCHEMES[scheme]
|
||||
return store_cls(uri)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
def _onsuccess(result):
|
||||
if not result:
|
||||
return # returning None force download
|
||||
|
|
@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline):
|
|||
checksum = result.get('checksum', None)
|
||||
return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
|
||||
|
||||
path = self.file_path(request, info=info)
|
||||
path = self.file_path(request, info=info, item=item)
|
||||
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
|
||||
dfd.addCallbacks(_onsuccess, lambda _: None)
|
||||
dfd.addErrback(
|
||||
|
|
@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline):
|
|||
|
||||
raise FileException
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
referer = referer_str(request)
|
||||
|
||||
if response.status != 200:
|
||||
|
|
@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline):
|
|||
self.inc_stats(info.spider, status)
|
||||
|
||||
try:
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
checksum = self.file_downloaded(response, request, info)
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
checksum = self.file_downloaded(response, request, info, item=item)
|
||||
except FileException as exc:
|
||||
logger.warning(
|
||||
'File (error): Error processing file from %(request)s '
|
||||
|
|
@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline):
|
|||
urls = ItemAdapter(item).get(self.files_urls_field, [])
|
||||
return [Request(u) for u in urls]
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
def file_downloaded(self, response, request, info, *, item=None):
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
buf = BytesIO(response.body)
|
||||
checksum = md5sum(buf)
|
||||
buf.seek(0)
|
||||
|
|
@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline):
|
|||
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
media_ext = os.path.splitext(request.url)[1]
|
||||
# Handles empty and wild extensions by trying to guess the
|
||||
|
|
|
|||
|
|
@ -45,8 +45,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
DEFAULT_IMAGES_RESULT_FIELD = 'images'
|
||||
|
||||
def __init__(self, store_uri, download_func=None, settings=None):
|
||||
super(ImagesPipeline, self).__init__(store_uri, settings=settings,
|
||||
download_func=download_func)
|
||||
super().__init__(store_uri, settings=settings, download_func=download_func)
|
||||
|
||||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
|
|
@ -104,12 +103,12 @@ class ImagesPipeline(FilesPipeline):
|
|||
store_uri = settings['IMAGES_STORE']
|
||||
return cls(store_uri, settings=settings)
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
return self.image_downloaded(response, request, info)
|
||||
def file_downloaded(self, response, request, info, *, item=None):
|
||||
return self.image_downloaded(response, request, info, item=item)
|
||||
|
||||
def image_downloaded(self, response, request, info):
|
||||
def image_downloaded(self, response, request, info, *, item=None):
|
||||
checksum = None
|
||||
for path, image, buf in self.get_images(response, request, info):
|
||||
for path, image, buf in self.get_images(response, request, info, item=item):
|
||||
if checksum is None:
|
||||
buf.seek(0)
|
||||
checksum = md5sum(buf)
|
||||
|
|
@ -120,8 +119,8 @@ class ImagesPipeline(FilesPipeline):
|
|||
headers={'Content-Type': 'image/jpeg'})
|
||||
return checksum
|
||||
|
||||
def get_images(self, response, request, info):
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
def get_images(self, response, request, info, *, item=None):
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
orig_image = Image.open(BytesIO(response.body))
|
||||
|
||||
width, height = orig_image.size
|
||||
|
|
@ -167,7 +166,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
return 'full/%s.jpg' % (image_guid)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
import functools
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from inspect import signature
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
|
@ -27,6 +31,7 @@ class MediaPipeline:
|
|||
|
||||
def __init__(self, download_func=None, settings=None):
|
||||
self.download_func = download_func
|
||||
self._expects_item = {}
|
||||
|
||||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
|
|
@ -38,6 +43,9 @@ class MediaPipeline:
|
|||
)
|
||||
self._handle_statuses(self.allow_redirects)
|
||||
|
||||
# Check if deprecated methods are being used and make them compatible
|
||||
self._make_compatible()
|
||||
|
||||
def _handle_statuses(self, allow_redirects):
|
||||
self.handle_httpstatus_list = None
|
||||
if allow_redirects:
|
||||
|
|
@ -77,11 +85,11 @@ class MediaPipeline:
|
|||
def process_item(self, item, spider):
|
||||
info = self.spiderinfo
|
||||
requests = arg_to_iter(self.get_media_requests(item, info))
|
||||
dlist = [self._process_request(r, info) for r in requests]
|
||||
dlist = [self._process_request(r, info, item) for r in requests]
|
||||
dfd = DeferredList(dlist, consumeErrors=1)
|
||||
return dfd.addCallback(self.item_completed, item, info)
|
||||
|
||||
def _process_request(self, request, info):
|
||||
def _process_request(self, request, info, item):
|
||||
fp = request_fingerprint(request)
|
||||
cb = request.callback or (lambda _: _)
|
||||
eb = request.errback
|
||||
|
|
@ -102,34 +110,73 @@ class MediaPipeline:
|
|||
|
||||
# Download request checking media_to_download hook output first
|
||||
info.downloading.add(fp)
|
||||
dfd = mustbe_deferred(self.media_to_download, request, info)
|
||||
dfd.addCallback(self._check_media_to_download, request, info)
|
||||
dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
|
||||
dfd.addCallback(self._check_media_to_download, request, info, item=item)
|
||||
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
|
||||
dfd.addErrback(lambda f: logger.error(
|
||||
f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider})
|
||||
)
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
|
||||
def _make_compatible(self):
|
||||
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
|
||||
methods = [
|
||||
"file_path", "media_to_download", "media_downloaded",
|
||||
"file_downloaded", "image_downloaded", "get_images"
|
||||
]
|
||||
|
||||
for method_name in methods:
|
||||
method = getattr(self, method_name, None)
|
||||
if callable(method):
|
||||
setattr(self, method_name, self._compatible(method))
|
||||
|
||||
def _compatible(self, func):
|
||||
"""Wrapper for overridable methods to allow backwards compatibility"""
|
||||
self._check_signature(func)
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if self._expects_item[func.__name__]:
|
||||
return func(*args, **kwargs)
|
||||
|
||||
kwargs.pop('item', None)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
def _check_signature(self, func):
|
||||
sig = signature(func)
|
||||
self._expects_item[func.__name__] = True
|
||||
|
||||
if 'item' not in sig.parameters:
|
||||
old_params = str(sig)[1:-1]
|
||||
new_params = old_params + ", *, item=None"
|
||||
warn('%s(self, %s) is deprecated, '
|
||||
'please use %s(self, %s)'
|
||||
% (func.__name__, old_params, func.__name__, new_params),
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._expects_item[func.__name__] = False
|
||||
|
||||
def _modify_media_request(self, request):
|
||||
if self.handle_httpstatus_list:
|
||||
request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
|
||||
else:
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
|
||||
def _check_media_to_download(self, result, request, info):
|
||||
def _check_media_to_download(self, result, request, info, item):
|
||||
if result is not None:
|
||||
return result
|
||||
if self.download_func:
|
||||
# this ugly code was left only to support tests. TODO: remove
|
||||
dfd = mustbe_deferred(self.download_func, request, info.spider)
|
||||
dfd.addCallbacks(
|
||||
callback=self.media_downloaded, callbackArgs=(request, info),
|
||||
callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
|
||||
errback=self.media_failed, errbackArgs=(request, info))
|
||||
else:
|
||||
self._modify_media_request(request)
|
||||
dfd = self.crawler.engine.download(request, info.spider)
|
||||
dfd.addCallbacks(
|
||||
callback=self.media_downloaded, callbackArgs=(request, info),
|
||||
callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
|
||||
errback=self.media_failed, errbackArgs=(request, info))
|
||||
return dfd
|
||||
|
||||
|
|
@ -171,7 +218,7 @@ class MediaPipeline:
|
|||
defer_result(result).chainDeferred(wad)
|
||||
|
||||
# Overridable Interface
|
||||
def media_to_download(self, request, info):
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
"""Check request before starting download"""
|
||||
pass
|
||||
|
||||
|
|
@ -179,7 +226,7 @@ class MediaPipeline:
|
|||
"""Returns the media requests to download"""
|
||||
pass
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
"""Handler for success downloads"""
|
||||
return response
|
||||
|
||||
|
|
@ -199,3 +246,7 @@ class MediaPipeline:
|
|||
extra={'spider': info.spider}
|
||||
)
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
"""Returns the path where downloaded media should be stored"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ class CachingThreadedResolver(ThreadedResolver):
|
|||
"""
|
||||
|
||||
def __init__(self, reactor, cache_size, timeout):
|
||||
super(CachingThreadedResolver, self).__init__(reactor)
|
||||
super().__init__(reactor)
|
||||
dnscache.limit = cache_size
|
||||
self.timeout = timeout
|
||||
|
||||
|
|
@ -40,7 +40,7 @@ class CachingThreadedResolver(ThreadedResolver):
|
|||
# so the input argument above is simply overridden
|
||||
# to enforce Scrapy's DNS_TIMEOUT setting's value
|
||||
timeout = (self.timeout,)
|
||||
d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
|
||||
d = super().getHostByName(name, timeout)
|
||||
if dnscache.limit:
|
||||
d.addCallback(self._cache_result, name)
|
||||
return d
|
||||
|
|
@ -80,16 +80,16 @@ class CachingHostnameResolver:
|
|||
class CachingResolutionReceiver(resolutionReceiver):
|
||||
|
||||
def resolutionBegan(self, resolution):
|
||||
super(CachingResolutionReceiver, self).resolutionBegan(resolution)
|
||||
super().resolutionBegan(resolution)
|
||||
self.resolution = resolution
|
||||
self.resolved = False
|
||||
|
||||
def addressResolved(self, address):
|
||||
super(CachingResolutionReceiver, self).addressResolved(address)
|
||||
super().addressResolved(address)
|
||||
self.resolved = True
|
||||
|
||||
def resolutionComplete(self):
|
||||
super(CachingResolutionReceiver, self).resolutionComplete()
|
||||
super().resolutionComplete()
|
||||
if self.resolved:
|
||||
dnscache[hostName] = self.resolution
|
||||
|
||||
|
|
|
|||
|
|
@ -47,10 +47,10 @@ class RobotParser(metaclass=ABCMeta):
|
|||
"""Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
|
||||
|
||||
:param url: Absolute URL
|
||||
:type url: string
|
||||
:type url: str
|
||||
|
||||
:param user_agent: User agent
|
||||
:type user_agent: string
|
||||
:type user_agent: str
|
||||
"""
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -79,4 +79,4 @@ class Selector(_ParselSelector, object_ref):
|
|||
kwargs.setdefault('base_url', response.url)
|
||||
|
||||
self.response = response
|
||||
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
|
||||
super().__init__(text=text, type=st, root=root, **kwargs)
|
||||
|
|
|
|||
|
|
@ -52,8 +52,7 @@ class SettingsAttribute:
|
|||
self.priority = priority
|
||||
|
||||
def __str__(self):
|
||||
return "<SettingsAttribute value={self.value!r} " \
|
||||
"priority={self.priority}>".format(self=self)
|
||||
return "<SettingsAttribute value={self.value!r} priority={self.priority}>".format(self=self)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
|
|
@ -83,7 +82,8 @@ class BaseSettings(MutableMapping):
|
|||
def __init__(self, values=None, priority='project'):
|
||||
self.frozen = False
|
||||
self.attributes = {}
|
||||
self.update(values, priority)
|
||||
if values:
|
||||
self.update(values, priority)
|
||||
|
||||
def __getitem__(self, opt_name):
|
||||
if opt_name not in self:
|
||||
|
|
@ -98,10 +98,10 @@ class BaseSettings(MutableMapping):
|
|||
Get a setting value without affecting its original type.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
return self[name] if self[name] is not None else default
|
||||
|
||||
|
|
@ -116,10 +116,10 @@ class BaseSettings(MutableMapping):
|
|||
``'0'`` will return ``False`` when using this method.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
got = self.get(name, default)
|
||||
try:
|
||||
|
|
@ -138,10 +138,10 @@ class BaseSettings(MutableMapping):
|
|||
Get a setting value as an int.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
return int(self.get(name, default))
|
||||
|
||||
|
|
@ -150,10 +150,10 @@ class BaseSettings(MutableMapping):
|
|||
Get a setting value as a float.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
return float(self.get(name, default))
|
||||
|
||||
|
|
@ -166,10 +166,10 @@ class BaseSettings(MutableMapping):
|
|||
``'one,two'`` will return a list ['one', 'two'] when using this method.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
value = self.get(name, default or [])
|
||||
if isinstance(value, str):
|
||||
|
|
@ -187,10 +187,10 @@ class BaseSettings(MutableMapping):
|
|||
and losing all information about priority and mutability.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
value = self.get(name, default or {})
|
||||
if isinstance(value, str):
|
||||
|
|
@ -202,7 +202,7 @@ class BaseSettings(MutableMapping):
|
|||
counterpart.
|
||||
|
||||
:param name: name of the dictionary-like setting
|
||||
:type name: string
|
||||
:type name: str
|
||||
"""
|
||||
compbs = BaseSettings()
|
||||
compbs.update(self[name + '_BASE'])
|
||||
|
|
@ -215,7 +215,7 @@ class BaseSettings(MutableMapping):
|
|||
the given ``name`` does not exist.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
"""
|
||||
if name not in self:
|
||||
return None
|
||||
|
|
@ -245,14 +245,14 @@ class BaseSettings(MutableMapping):
|
|||
otherwise they won't have any effect.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param value: the value to associate with the setting
|
||||
:type value: any
|
||||
:type value: object
|
||||
|
||||
:param priority: the priority of the setting. Should be a key of
|
||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||
:type priority: string or int
|
||||
:type priority: str or int
|
||||
"""
|
||||
self._assert_mutability()
|
||||
priority = get_settings_priority(priority)
|
||||
|
|
@ -276,11 +276,11 @@ class BaseSettings(MutableMapping):
|
|||
uppercase variable of ``module`` with the provided ``priority``.
|
||||
|
||||
:param module: the module or the path of the module
|
||||
:type module: module object or string
|
||||
:type module: types.ModuleType or str
|
||||
|
||||
:param priority: the priority of the settings. Should be a key of
|
||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||
:type priority: string or int
|
||||
:type priority: str or int
|
||||
"""
|
||||
self._assert_mutability()
|
||||
if isinstance(module, str):
|
||||
|
|
@ -309,7 +309,7 @@ class BaseSettings(MutableMapping):
|
|||
|
||||
:param priority: the priority of the settings. Should be a key of
|
||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||
:type priority: string or int
|
||||
:type priority: str or int
|
||||
"""
|
||||
self._assert_mutability()
|
||||
if isinstance(values, str):
|
||||
|
|
@ -440,7 +440,7 @@ class Settings(BaseSettings):
|
|||
# Do not pass kwarg values here. We don't want to promote user-defined
|
||||
# dicts, and we want to update, not replace, default dicts with the
|
||||
# values given by the user
|
||||
super(Settings, self).__init__()
|
||||
super().__init__()
|
||||
self.setmodule(default_settings, 'default')
|
||||
# Promote default dictionaries to BaseSettings instances for per-key
|
||||
# priorities
|
||||
|
|
|
|||
|
|
@ -142,10 +142,12 @@ FEED_STORAGES = {}
|
|||
FEED_STORAGES_BASE = {
|
||||
'': 'scrapy.extensions.feedexport.FileFeedStorage',
|
||||
'file': 'scrapy.extensions.feedexport.FileFeedStorage',
|
||||
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
|
||||
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
|
||||
'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
|
||||
'gs': 'scrapy.extensions.feedexport.GCSFeedStorage',
|
||||
's3': 'scrapy.extensions.feedexport.S3FeedStorage',
|
||||
'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
|
||||
}
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT = 0
|
||||
FEED_EXPORTERS = {}
|
||||
FEED_EXPORTERS_BASE = {
|
||||
'json': 'scrapy.exporters.JsonItemExporter',
|
||||
|
|
@ -159,6 +161,7 @@ FEED_EXPORTERS_BASE = {
|
|||
FEED_EXPORT_INDENT = 0
|
||||
|
||||
FEED_STORAGE_FTP_ACTIVE = False
|
||||
FEED_STORAGE_GCS_ACL = ''
|
||||
FEED_STORAGE_S3_ACL = ''
|
||||
|
||||
FILES_STORE_S3_ACL = 'private'
|
||||
|
|
@ -168,6 +171,8 @@ FTP_USER = 'anonymous'
|
|||
FTP_PASSWORD = 'guest'
|
||||
FTP_PASSIVE_MODE = True
|
||||
|
||||
GCS_PROJECT_ID = None
|
||||
|
||||
HTTPCACHE_ENABLED = False
|
||||
HTTPCACHE_DIR = 'httpcache'
|
||||
HTTPCACHE_IGNORE_MISSING = False
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class SignalManager:
|
|||
section.
|
||||
|
||||
:param receiver: the function to be connected
|
||||
:type receiver: callable
|
||||
:type receiver: collections.abc.Callable
|
||||
|
||||
:param signal: the signal to connect to
|
||||
:type signal: object
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class HttpError(IgnoreRequest):
|
|||
|
||||
def __init__(self, response, *args, **kwargs):
|
||||
self.response = response
|
||||
super(HttpError, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class HttpErrorMiddleware:
|
||||
|
|
|
|||
|
|
@ -86,7 +86,10 @@ class Spider(object_ref):
|
|||
)
|
||||
return Request(url, dont_filter=True)
|
||||
|
||||
def parse(self, response):
|
||||
def _parse(self, response, **kwargs):
|
||||
return self.parse(response, **kwargs)
|
||||
|
||||
def parse(self, response, **kwargs):
|
||||
raise NotImplementedError('{}.parse callback is not defined'.format(self.__class__.__name__))
|
||||
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -75,13 +75,18 @@ class CrawlSpider(Spider):
|
|||
rules = ()
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(CrawlSpider, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
self._compile_rules()
|
||||
|
||||
def parse(self, response):
|
||||
return self._parse_response(response, self.parse_start_url, cb_kwargs={}, follow=True)
|
||||
def _parse(self, response, **kwargs):
|
||||
return self._parse_response(
|
||||
response=response,
|
||||
callback=self.parse_start_url,
|
||||
cb_kwargs=kwargs,
|
||||
follow=True,
|
||||
)
|
||||
|
||||
def parse_start_url(self, response):
|
||||
def parse_start_url(self, response, **kwargs):
|
||||
return []
|
||||
|
||||
def process_results(self, response, results):
|
||||
|
|
@ -140,6 +145,6 @@ class CrawlSpider(Spider):
|
|||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
||||
spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
|
||||
return spider
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class XMLFeedSpider(Spider):
|
|||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
||||
def parse(self, response):
|
||||
def _parse(self, response, **kwargs):
|
||||
if not hasattr(self, 'parse_node'):
|
||||
raise NotConfigured('You must define parse_node method in order to scrape this XML feed')
|
||||
|
||||
|
|
@ -128,7 +128,7 @@ class CSVFeedSpider(Spider):
|
|||
for result_item in self.process_results(response, ret):
|
||||
yield result_item
|
||||
|
||||
def parse(self, response):
|
||||
def _parse(self, response, **kwargs):
|
||||
if not hasattr(self, 'parse_row'):
|
||||
raise NotConfigured('You must define parse_row method in order to scrape this CSV feed')
|
||||
response = self.adapt_response(response)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ class InitSpider(Spider):
|
|||
"""Base Spider with initialization facilities"""
|
||||
|
||||
def start_requests(self):
|
||||
self._postinit_reqs = super(InitSpider, self).start_requests()
|
||||
self._postinit_reqs = super().start_requests()
|
||||
return iterate_spider_output(self.init_request())
|
||||
|
||||
def initialized(self, response=None):
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ class SitemapSpider(Spider):
|
|||
sitemap_alternate_links = False
|
||||
|
||||
def __init__(self, *a, **kw):
|
||||
super(SitemapSpider, self).__init__(*a, **kw)
|
||||
super().__init__(*a, **kw)
|
||||
self._cbs = []
|
||||
for r, c in self.sitemap_rules:
|
||||
if isinstance(c, str):
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def _with_mkdir(queue_class):
|
|||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
|
||||
super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
|
||||
super().__init__(path, *args, **kwargs)
|
||||
|
||||
return DirectoriesCreated
|
||||
|
||||
|
|
@ -31,10 +31,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
|
|||
|
||||
def push(self, obj):
|
||||
s = serialize(obj)
|
||||
super(SerializableQueue, self).push(s)
|
||||
super().push(s)
|
||||
|
||||
def pop(self):
|
||||
s = super(SerializableQueue, self).pop()
|
||||
s = super().pop()
|
||||
if s:
|
||||
return deserialize(s)
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ def _scrapy_serialization_queue(queue_class):
|
|||
|
||||
def __init__(self, crawler, key):
|
||||
self.spider = crawler.spider
|
||||
super(ScrapyRequestQueue, self).__init__(key)
|
||||
super().__init__(key)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, key, *args, **kwargs):
|
||||
|
|
@ -55,10 +55,10 @@ def _scrapy_serialization_queue(queue_class):
|
|||
|
||||
def push(self, request):
|
||||
request = request_to_dict(request, self.spider)
|
||||
return super(ScrapyRequestQueue, self).push(request)
|
||||
return super().push(request)
|
||||
|
||||
def pop(self):
|
||||
request = super(ScrapyRequestQueue, self).pop()
|
||||
request = super().pop()
|
||||
|
||||
if not request:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -54,7 +54,7 @@ class StatsCollector:
|
|||
class MemoryStatsCollector(StatsCollector):
|
||||
|
||||
def __init__(self, crawler):
|
||||
super(MemoryStatsCollector, self).__init__(crawler)
|
||||
super().__init__(crawler)
|
||||
self.spider_stats = {}
|
||||
|
||||
def _persist_stats(self, stats, spider):
|
||||
|
|
|
|||
|
|
@ -28,8 +28,7 @@ class Root(Resource):
|
|||
|
||||
|
||||
def _getarg(request, name, default=None, type=str):
|
||||
return type(request.args[name][0]) \
|
||||
if name in request.args else default
|
||||
return type(request.args[name][0]) if name in request.args else default
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -101,11 +101,13 @@ def get_config(use_closest=True):
|
|||
|
||||
|
||||
def get_sources(use_closest=True):
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \
|
||||
os.path.expanduser('~/.config')
|
||||
sources = ['/etc/scrapy.cfg', r'c:\scrapy\scrapy.cfg',
|
||||
xdg_config_home + '/scrapy.cfg',
|
||||
os.path.expanduser('~/.scrapy.cfg')]
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.expanduser('~/.config')
|
||||
sources = [
|
||||
'/etc/scrapy.cfg',
|
||||
r'c:\scrapy\scrapy.cfg',
|
||||
xdg_config_home + '/scrapy.cfg',
|
||||
os.path.expanduser('~/.scrapy.cfg'),
|
||||
]
|
||||
if use_closest:
|
||||
sources.append(closest_scrapy_cfg())
|
||||
return sources
|
||||
|
|
@ -113,6 +115,7 @@ def get_sources(use_closest=True):
|
|||
|
||||
def feed_complete_default_values_from_settings(feed, settings):
|
||||
out = feed.copy()
|
||||
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
|
||||
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
|
||||
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ from w3lib.http import basic_auth_header
|
|||
|
||||
class CurlParser(argparse.ArgumentParser):
|
||||
def error(self, message):
|
||||
error_msg = \
|
||||
'There was an error parsing the curl command: {}'.format(message)
|
||||
error_msg = 'There was an error parsing the curl command: {}'.format(message)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
|
||||
|
|
@ -40,7 +39,8 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
|
|||
|
||||
:param str curl_command: string containing the curl command
|
||||
:param bool ignore_unknown_options: If true, only a warning is emitted when
|
||||
cURL options are unknown. Otherwise raises an error. (default: True)
|
||||
cURL options are unknown. Otherwise
|
||||
raises an error. (default: True)
|
||||
:return: dictionary of Request kwargs
|
||||
"""
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class CaselessDict(dict):
|
|||
__slots__ = ()
|
||||
|
||||
def __init__(self, seq=None):
|
||||
super(CaselessDict, self).__init__()
|
||||
super().__init__()
|
||||
if seq:
|
||||
self.update(seq)
|
||||
|
||||
|
|
@ -53,7 +53,7 @@ class CaselessDict(dict):
|
|||
def update(self, seq):
|
||||
seq = seq.items() if isinstance(seq, Mapping) else seq
|
||||
iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
|
||||
super(CaselessDict, self).update(iseq)
|
||||
super().update(iseq)
|
||||
|
||||
@classmethod
|
||||
def fromkeys(cls, keys, value=None):
|
||||
|
|
@ -70,14 +70,14 @@ class LocalCache(collections.OrderedDict):
|
|||
"""
|
||||
|
||||
def __init__(self, limit=None):
|
||||
super(LocalCache, self).__init__()
|
||||
super().__init__()
|
||||
self.limit = limit
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
if self.limit:
|
||||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
super(LocalCache, self).__setitem__(key, value)
|
||||
super().__setitem__(key, value)
|
||||
|
||||
|
||||
class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
|
||||
|
|
@ -93,18 +93,18 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
|
|||
"""
|
||||
|
||||
def __init__(self, limit=None):
|
||||
super(LocalWeakReferencedCache, self).__init__()
|
||||
super().__init__()
|
||||
self.data = LocalCache(limit=limit)
|
||||
|
||||
def __setitem__(self, key, value):
|
||||
try:
|
||||
super(LocalWeakReferencedCache, self).__setitem__(key, value)
|
||||
super().__setitem__(key, value)
|
||||
except TypeError:
|
||||
pass # key is not weak-referenceable, skip caching
|
||||
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return super(LocalWeakReferencedCache, self).__getitem__(key)
|
||||
return super().__getitem__(key)
|
||||
except (TypeError, KeyError):
|
||||
return None # key is either not weak-referenceable or not cached
|
||||
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ def create_deprecated_class(
|
|||
warned_on_subclass = False
|
||||
|
||||
def __new__(metacls, name, bases, clsdict_):
|
||||
cls = super(DeprecatedClass, metacls).__new__(metacls, name, bases, clsdict_)
|
||||
cls = super().__new__(metacls, name, bases, clsdict_)
|
||||
if metacls.deprecated_class is None:
|
||||
metacls.deprecated_class = cls
|
||||
return cls
|
||||
|
|
@ -73,7 +73,7 @@ def create_deprecated_class(
|
|||
if warn_once:
|
||||
msg += ' (warning only on first subclass, there may be others)'
|
||||
warnings.warn(msg, warn_category, stacklevel=2)
|
||||
super(DeprecatedClass, cls).__init__(name, bases, clsdict_)
|
||||
super().__init__(name, bases, clsdict_)
|
||||
|
||||
# see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass
|
||||
# and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks
|
||||
|
|
@ -88,7 +88,7 @@ def create_deprecated_class(
|
|||
# is the deprecated class itself - subclasses of the
|
||||
# deprecated class should not use custom `__subclasscheck__`
|
||||
# method.
|
||||
return super(DeprecatedClass, cls).__subclasscheck__(sub)
|
||||
return super().__subclasscheck__(sub)
|
||||
|
||||
if not inspect.isclass(sub):
|
||||
raise TypeError("issubclass() arg 1 must be a class")
|
||||
|
|
@ -102,7 +102,7 @@ def create_deprecated_class(
|
|||
msg = instance_warn_message.format(cls=_clspath(cls, old_class_path),
|
||||
new=_clspath(new_class, new_class_path))
|
||||
warnings.warn(msg, warn_category, stacklevel=2)
|
||||
return super(DeprecatedClass, cls).__call__(*args, **kwargs)
|
||||
return super().__call__(*args, **kwargs)
|
||||
|
||||
deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})
|
||||
|
||||
|
|
|
|||
|
|
@ -2,20 +2,42 @@
|
|||
pprint and pformat wrappers with colorization support
|
||||
"""
|
||||
|
||||
import ctypes
|
||||
import platform
|
||||
import sys
|
||||
from distutils.version import LooseVersion as parse_version
|
||||
from pprint import pformat as pformat_
|
||||
|
||||
|
||||
def _enable_windows_terminal_processing():
|
||||
# https://stackoverflow.com/a/36760881
|
||||
kernel32 = ctypes.windll.kernel32
|
||||
return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
|
||||
|
||||
|
||||
def _tty_supports_color():
|
||||
if sys.platform != "win32":
|
||||
return True
|
||||
|
||||
if parse_version(platform.version()) < parse_version("10.0.14393"):
|
||||
return True
|
||||
|
||||
# Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
|
||||
# processing is enabled.
|
||||
return _enable_windows_terminal_processing()
|
||||
|
||||
|
||||
def _colorize(text, colorize=True):
|
||||
if not colorize or not sys.stdout.isatty():
|
||||
if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
|
||||
return text
|
||||
try:
|
||||
from pygments import highlight
|
||||
except ImportError:
|
||||
return text
|
||||
else:
|
||||
from pygments.formatters import TerminalFormatter
|
||||
from pygments.lexers import PythonLexer
|
||||
return highlight(text, PythonLexer(), TerminalFormatter())
|
||||
except ImportError:
|
||||
return text
|
||||
|
||||
|
||||
def pformat(obj, *args, **kwargs):
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ class LogCounterHandler(logging.Handler):
|
|||
"""Record log levels count into a crawler stats"""
|
||||
|
||||
def __init__(self, crawler, *args, **kwargs):
|
||||
super(LogCounterHandler, self).__init__(*args, **kwargs)
|
||||
super().__init__(*args, **kwargs)
|
||||
self.crawler = crawler
|
||||
|
||||
def emit(self, record):
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from w3lib.html import replace_entities
|
|||
from scrapy.utils.datatypes import LocalWeakReferencedCache
|
||||
from scrapy.utils.python import flatten, to_unicode
|
||||
from scrapy.item import _BaseItem
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
|
||||
|
|
@ -86,6 +87,11 @@ def extract_regex(regex, text, encoding='utf-8'):
|
|||
* if the regex contains multiple numbered groups, all those will be returned (flattened)
|
||||
* if the regex doesn't contain any group the entire regex matching is returned
|
||||
"""
|
||||
warnings.warn(
|
||||
"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
|
||||
ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
|
||||
if isinstance(regex, str):
|
||||
regex = re.compile(regex, re.UNICODE)
|
||||
|
|
|
|||
|
|
@ -18,8 +18,7 @@ def install_shutdown_handlers(function, override_sigint=True):
|
|||
from twisted.internet import reactor
|
||||
reactor._handleSignals()
|
||||
signal.signal(signal.SIGTERM, function)
|
||||
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or \
|
||||
override_sigint:
|
||||
if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
|
||||
signal.signal(signal.SIGINT, function)
|
||||
# Catch Ctrl-Break in windows
|
||||
if hasattr(signal, 'SIGBREAK'):
|
||||
|
|
|
|||
|
|
@ -6,10 +6,12 @@ import gc
|
|||
import inspect
|
||||
import re
|
||||
import sys
|
||||
import warnings
|
||||
import weakref
|
||||
from functools import partial, wraps
|
||||
from itertools import chain
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.decorators import deprecated
|
||||
|
||||
|
||||
|
|
@ -127,6 +129,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
|
|||
In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing
|
||||
the start position of the match, and the ending (regarding the entire text).
|
||||
"""
|
||||
|
||||
def _chunk_iter():
|
||||
offset = len(text)
|
||||
while True:
|
||||
|
|
@ -158,6 +161,7 @@ def memoizemethod_noargs(method):
|
|||
if self not in cache:
|
||||
cache[self] = method(self, *args, **kwargs)
|
||||
return cache[self]
|
||||
|
||||
return new_method
|
||||
|
||||
|
||||
|
|
@ -276,6 +280,7 @@ def equal_attributes(obj1, obj2, attributes):
|
|||
class WeakKeyCache:
|
||||
|
||||
def __init__(self, default_factory):
|
||||
warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
self.default_factory = default_factory
|
||||
self._weakdict = weakref.WeakKeyDictionary()
|
||||
|
||||
|
|
@ -285,6 +290,7 @@ class WeakKeyCache:
|
|||
return self._weakdict[key]
|
||||
|
||||
|
||||
@deprecated
|
||||
def retry_on_eintr(function, *args, **kw):
|
||||
"""Run a function and retry it while getting EINTR errors"""
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -47,13 +47,17 @@ def response_httprepr(response):
|
|||
is provided only for reference, since it's not the exact stream of bytes
|
||||
that was received (that's not exposed by Twisted).
|
||||
"""
|
||||
s = b"HTTP/1.1 " + to_bytes(str(response.status)) + b" " + \
|
||||
to_bytes(http.RESPONSES.get(response.status, b'')) + b"\r\n"
|
||||
values = [
|
||||
b"HTTP/1.1 ",
|
||||
to_bytes(str(response.status)),
|
||||
b" ",
|
||||
to_bytes(http.RESPONSES.get(response.status, b'')),
|
||||
b"\r\n",
|
||||
]
|
||||
if response.headers:
|
||||
s += response.headers.to_string() + b"\r\n"
|
||||
s += b"\r\n"
|
||||
s += response.body
|
||||
return s
|
||||
values.extend([response.headers.to_string(), b"\r\n"])
|
||||
values.extend([b"\r\n", response.body])
|
||||
return b"".join(values)
|
||||
|
||||
|
||||
def open_in_browser(response, _openfunc=webbrowser.open):
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class ScrapyJSONEncoder(json.JSONEncoder):
|
|||
elif isinstance(o, Response):
|
||||
return "<%s %s %s>" % (type(o).__name__, o.status, o.url)
|
||||
else:
|
||||
return super(ScrapyJSONEncoder, self).default(o)
|
||||
return super().default(o)
|
||||
|
||||
|
||||
class ScrapyJSONDecoder(json.JSONDecoder):
|
||||
|
|
|
|||
|
|
@ -34,10 +34,12 @@ def iter_spider_classes(module):
|
|||
from scrapy.spiders import Spider
|
||||
|
||||
for obj in vars(module).values():
|
||||
if inspect.isclass(obj) and \
|
||||
issubclass(obj, Spider) and \
|
||||
obj.__module__ == module.__name__ and \
|
||||
getattr(obj, 'name', None):
|
||||
if (
|
||||
inspect.isclass(obj)
|
||||
and issubclass(obj, Spider)
|
||||
and obj.__module__ == module.__name__
|
||||
and getattr(obj, 'name', None)
|
||||
):
|
||||
yield obj
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@
|
|||
This module contains some assorted functions used in tests
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import
|
||||
from posixpath import split
|
||||
import asyncio
|
||||
import os
|
||||
from posixpath import split
|
||||
from unittest import mock
|
||||
|
||||
from importlib import import_module
|
||||
from twisted.trial.unittest import SkipTest
|
||||
|
|
@ -126,3 +126,19 @@ def get_from_asyncio_queue(value):
|
|||
getter = q.get()
|
||||
q.put_nowait(value)
|
||||
return getter
|
||||
|
||||
|
||||
def mock_google_cloud_storage():
|
||||
"""Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
|
||||
classes and set their proper return values.
|
||||
"""
|
||||
from google.cloud.storage import Client, Bucket, Blob
|
||||
client_mock = mock.create_autospec(Client)
|
||||
|
||||
bucket_mock = mock.create_autospec(Bucket)
|
||||
client_mock.get_bucket.return_value = bucket_mock
|
||||
|
||||
blob_mock = mock.create_autospec(Blob)
|
||||
bucket_mock.blob.return_value = blob_mock
|
||||
|
||||
return (client_mock, bucket_mock, blob_mock)
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue