mirror of https://github.com/scrapy/scrapy.git
Merge remote-tracking branch 'upstream/master' into tox-pip20.2
This commit is contained in:
Adrián Chaves
commit
0567fdc56e
|
|
@ -1,5 +1,5 @@
|
|||
[bumpversion]
|
||||
current_version = 2.3.0
|
||||
current_version = 2.4.1
|
||||
commit = True
|
||||
tag = True
|
||||
tag_name = {new_version}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
name: Run test suite
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
test-windows:
|
||||
name: "Windows Tests"
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
os: [windows-latest]
|
||||
python-version: [3.7, 3.8]
|
||||
env: [TOXENV: py]
|
||||
include:
|
||||
- os: windows-latest
|
||||
python-version: 3.6
|
||||
env:
|
||||
TOXENV: windows-pinned
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v1
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Run test suite
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox twine wheel codecov
|
||||
tox
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
variables:
|
||||
TOXENV: py
|
||||
pool:
|
||||
vmImage: 'windows-latest'
|
||||
strategy:
|
||||
matrix:
|
||||
Python36:
|
||||
python.version: '3.6'
|
||||
TOXENV: windows-pinned
|
||||
Python37:
|
||||
python.version: '3.7'
|
||||
Python38:
|
||||
python.version: '3.8'
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '$(python.version)'
|
||||
displayName: 'Use Python $(python.version)'
|
||||
- script: |
|
||||
pip install -U tox twine wheel codecov
|
||||
tox
|
||||
displayName: 'Run test suite'
|
||||
|
|
@ -283,6 +283,7 @@ coverage_ignore_pyobjects = [
|
|||
intersphinx_mapping = {
|
||||
'attrs': ('https://www.attrs.org/en/stable/', None),
|
||||
'coverage': ('https://coverage.readthedocs.io/en/stable', None),
|
||||
'cryptography' : ('https://cryptography.io/en/latest/', None),
|
||||
'cssselect': ('https://cssselect.readthedocs.io/en/latest', None),
|
||||
'itemloaders': ('https://itemloaders.readthedocs.io/en/latest/', None),
|
||||
'pytest': ('https://docs.pytest.org/en/latest', None),
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ original pull request author hasn't had time to address them.
|
|||
In this case consider picking up this pull request: open
|
||||
a new pull request with all commits from the original pull request, as well as
|
||||
additional changes to address the raised issues. Doing so helps a lot; it is
|
||||
not considered rude as soon as the original author is acknowledged by keeping
|
||||
not considered rude as long as the original author is acknowledged by keeping
|
||||
his/her commits.
|
||||
|
||||
You can pull an existing pull request to a local branch
|
||||
|
|
|
|||
|
|
@ -78,7 +78,6 @@ Basic concepts
|
|||
topics/settings
|
||||
topics/exceptions
|
||||
|
||||
|
||||
:doc:`topics/commands`
|
||||
Learn about the command-line tool used to manage your Scrapy project.
|
||||
|
||||
|
|
|
|||
|
|
@ -69,10 +69,9 @@ In case of any trouble related to these dependencies,
|
|||
please refer to their respective installation instructions:
|
||||
|
||||
* `lxml installation`_
|
||||
* `cryptography installation`_
|
||||
* :doc:`cryptography installation <cryptography:installation>`
|
||||
|
||||
.. _lxml installation: https://lxml.de/installation.html
|
||||
.. _cryptography installation: https://cryptography.io/en/latest/installation/
|
||||
|
||||
|
||||
.. _intro-using-virtualenv:
|
||||
|
|
@ -265,7 +264,6 @@ For details, see `Issue #2473 <https://github.com/scrapy/scrapy/issues/2473>`_.
|
|||
.. _cryptography: https://cryptography.io/en/latest/
|
||||
.. _pyOpenSSL: https://pypi.org/project/pyOpenSSL/
|
||||
.. _setuptools: https://pypi.python.org/pypi/setuptools
|
||||
.. _AUR Scrapy package: https://aur.archlinux.org/packages/scrapy/
|
||||
.. _homebrew: https://brew.sh/
|
||||
.. _zsh: https://www.zsh.org/
|
||||
.. _Scrapinghub: https://scrapinghub.com
|
||||
|
|
|
|||
329
docs/news.rst
329
docs/news.rst
|
|
@ -3,6 +3,333 @@
|
|||
Release notes
|
||||
=============
|
||||
|
||||
.. _release-2.4.1:
|
||||
|
||||
Scrapy 2.4.1 (2020-11-17)
|
||||
-------------------------
|
||||
|
||||
- Fixed :ref:`feed exports <topics-feed-exports>` overwrite support (:issue:`4845`, :issue:`4857`, :issue:`4859`)
|
||||
|
||||
- Fixed the AsyncIO event loop handling, which could make code hang
|
||||
(:issue:`4855`, :issue:`4872`)
|
||||
|
||||
- Fixed the IPv6-capable DNS resolver
|
||||
:class:`~scrapy.resolver.CachingHostnameResolver` for download handlers
|
||||
that call
|
||||
:meth:`reactor.resolve <twisted.internet.interfaces.IReactorCore.resolve>`
|
||||
(:issue:`4802`, :issue:`4803`)
|
||||
|
||||
- Fixed the output of the :command:`genspider` command showing placeholders
|
||||
instead of the import path of the generated spider module (:issue:`4874`)
|
||||
|
||||
- Migrated Windows CI from Azure Pipelines to GitHub Actions (:issue:`4869`,
|
||||
:issue:`4876`)
|
||||
|
||||
|
||||
.. _release-2.4.0:
|
||||
|
||||
Scrapy 2.4.0 (2020-10-11)
|
||||
-------------------------
|
||||
|
||||
Highlights:
|
||||
|
||||
* Python 3.5 support has been dropped.
|
||||
|
||||
* The ``file_path`` method of :ref:`media pipelines <topics-media-pipeline>`
|
||||
can now access the source :ref:`item <topics-items>`.
|
||||
|
||||
This allows you to set a download file path based on item data.
|
||||
|
||||
* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
|
||||
to define keyword parameters to pass to :ref:`item exporter classes
|
||||
<topics-exporters>`
|
||||
|
||||
* You can now choose whether :ref:`feed exports <topics-feed-exports>`
|
||||
overwrite or append to the output file.
|
||||
|
||||
For example, when using the :command:`crawl` or :command:`runspider`
|
||||
commands, you can use the ``-O`` option instead of ``-o`` to overwrite the
|
||||
output file.
|
||||
|
||||
* Zstd-compressed responses are now supported if zstandard_ is installed.
|
||||
|
||||
* In settings, where the import path of a class is required, it is now
|
||||
possible to pass a class object instead.
|
||||
|
||||
Modified requirements
|
||||
~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Python 3.6 or greater is now required; support for Python 3.5 has been
|
||||
dropped
|
||||
|
||||
As a result:
|
||||
|
||||
- When using PyPy, PyPy 7.2.0 or greater :ref:`is now required
|
||||
<faq-python-versions>`
|
||||
|
||||
- For Amazon S3 storage support in :ref:`feed exports
|
||||
<topics-feed-storage-s3>` or :ref:`media pipelines
|
||||
<media-pipelines-s3>`, botocore_ 1.4.87 or greater is now required
|
||||
|
||||
- To use the :ref:`images pipeline <images-pipeline>`, Pillow_ 4.0.0 or
|
||||
greater is now required
|
||||
|
||||
(:issue:`4718`, :issue:`4732`, :issue:`4733`, :issue:`4742`, :issue:`4743`,
|
||||
:issue:`4764`)
|
||||
|
||||
|
||||
Backward-incompatible changes
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware` once again
|
||||
discards cookies defined in :attr:`Request.headers
|
||||
<scrapy.http.Request.headers>`.
|
||||
|
||||
We decided to revert this bug fix, introduced in Scrapy 2.2.0, because it
|
||||
was reported that the current implementation could break existing code.
|
||||
|
||||
If you need to set cookies for a request, use the :class:`Request.cookies
|
||||
<scrapy.http.Request>` parameter.
|
||||
|
||||
A future version of Scrapy will include a new, better implementation of the
|
||||
reverted bug fix.
|
||||
|
||||
(:issue:`4717`, :issue:`4823`)
|
||||
|
||||
|
||||
Deprecation removals
|
||||
~~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
* :class:`scrapy.extensions.feedexport.S3FeedStorage` no longer reads the
|
||||
values of ``access_key`` and ``secret_key`` from the running project
|
||||
settings when they are not passed to its ``__init__`` method; you must
|
||||
either pass those parameters to its ``__init__`` method or use
|
||||
:class:`S3FeedStorage.from_crawler
|
||||
<scrapy.extensions.feedexport.S3FeedStorage.from_crawler>`
|
||||
(:issue:`4356`, :issue:`4411`, :issue:`4688`)
|
||||
|
||||
* :attr:`Rule.process_request <scrapy.spiders.crawl.Rule.process_request>`
|
||||
no longer admits callables which expect a single ``request`` parameter,
|
||||
rather than both ``request`` and ``response`` (:issue:`4818`)
|
||||
|
||||
|
||||
Deprecations
|
||||
~~~~~~~~~~~~
|
||||
|
||||
* In custom :ref:`media pipelines <topics-media-pipeline>`, signatures that
|
||||
do not accept a keyword-only ``item`` parameter in any of the methods that
|
||||
:ref:`now support this parameter <media-pipeline-item-parameter>` are now
|
||||
deprecated (:issue:`4628`, :issue:`4686`)
|
||||
|
||||
* In custom :ref:`feed storage backend classes <topics-feed-storage>`,
|
||||
``__init__`` method signatures that do not accept a keyword-only
|
||||
``feed_options`` parameter are now deprecated (:issue:`547`, :issue:`716`,
|
||||
:issue:`4512`)
|
||||
|
||||
* The :class:`scrapy.utils.python.WeakKeyCache` class is now deprecated
|
||||
(:issue:`4684`, :issue:`4701`)
|
||||
|
||||
* The :func:`scrapy.utils.boto.is_botocore` function is now deprecated, use
|
||||
:func:`scrapy.utils.boto.is_botocore_available` instead (:issue:`4734`,
|
||||
:issue:`4776`)
|
||||
|
||||
|
||||
New features
|
||||
~~~~~~~~~~~~
|
||||
|
||||
.. _media-pipeline-item-parameter:
|
||||
|
||||
* The following methods of :ref:`media pipelines <topics-media-pipeline>` now
|
||||
accept an ``item`` keyword-only parameter containing the source
|
||||
:ref:`item <topics-items>`:
|
||||
|
||||
- In :class:`scrapy.pipelines.files.FilesPipeline`:
|
||||
|
||||
- :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded`
|
||||
|
||||
- :meth:`~scrapy.pipelines.files.FilesPipeline.file_path`
|
||||
|
||||
- :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`
|
||||
|
||||
- :meth:`~scrapy.pipelines.files.FilesPipeline.media_to_download`
|
||||
|
||||
- In :class:`scrapy.pipelines.images.ImagesPipeline`:
|
||||
|
||||
- :meth:`~scrapy.pipelines.images.ImagesPipeline.file_downloaded`
|
||||
|
||||
- :meth:`~scrapy.pipelines.images.ImagesPipeline.file_path`
|
||||
|
||||
- :meth:`~scrapy.pipelines.images.ImagesPipeline.get_images`
|
||||
|
||||
- :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded`
|
||||
|
||||
- :meth:`~scrapy.pipelines.images.ImagesPipeline.media_downloaded`
|
||||
|
||||
- :meth:`~scrapy.pipelines.images.ImagesPipeline.media_to_download`
|
||||
|
||||
(:issue:`4628`, :issue:`4686`)
|
||||
|
||||
* The new ``item_export_kwargs`` key of the :setting:`FEEDS` setting allows
|
||||
to define keyword parameters to pass to :ref:`item exporter classes
|
||||
<topics-exporters>` (:issue:`4606`, :issue:`4768`)
|
||||
|
||||
* :ref:`Feed exports <topics-feed-exports>` gained overwrite support:
|
||||
|
||||
* When using the :command:`crawl` or :command:`runspider` commands, you
|
||||
can use the ``-O`` option instead of ``-o`` to overwrite the output
|
||||
file
|
||||
|
||||
* You can use the ``overwrite`` key in the :setting:`FEEDS` setting to
|
||||
configure whether to overwrite the output file (``True``) or append to
|
||||
its content (``False``)
|
||||
|
||||
* The ``__init__`` and ``from_crawler`` methods of :ref:`feed storage
|
||||
backend classes <topics-feed-storage>` now receive a new keyword-only
|
||||
parameter, ``feed_options``, which is a dictionary of :ref:`feed
|
||||
options <feed-options>`
|
||||
|
||||
(:issue:`547`, :issue:`716`, :issue:`4512`)
|
||||
|
||||
* Zstd-compressed responses are now supported if zstandard_ is installed
|
||||
(:issue:`4831`)
|
||||
|
||||
* In settings, where the import path of a class is required, it is now
|
||||
possible to pass a class object instead (:issue:`3870`, :issue:`3873`).
|
||||
|
||||
This includes also settings where only part of its value is made of an
|
||||
import path, such as :setting:`DOWNLOADER_MIDDLEWARES` or
|
||||
:setting:`DOWNLOAD_HANDLERS`.
|
||||
|
||||
* :ref:`Downloader middlewares <topics-downloader-middleware>` can now
|
||||
override :class:`response.request <scrapy.http.Response.request>`.
|
||||
|
||||
If a :ref:`downloader middleware <topics-downloader-middleware>` returns
|
||||
a :class:`~scrapy.http.Response` object from
|
||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
|
||||
or
|
||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`
|
||||
with a custom :class:`~scrapy.http.Request` object assigned to
|
||||
:class:`response.request <scrapy.http.Response.request>`:
|
||||
|
||||
- The response is handled by the callback of that custom
|
||||
:class:`~scrapy.http.Request` object, instead of being handled by the
|
||||
callback of the original :class:`~scrapy.http.Request` object
|
||||
|
||||
- That custom :class:`~scrapy.http.Request` object is now sent as the
|
||||
``request`` argument to the :signal:`response_received` signal, instead
|
||||
of the original :class:`~scrapy.http.Request` object
|
||||
|
||||
(:issue:`4529`, :issue:`4632`)
|
||||
|
||||
* When using the :ref:`FTP feed storage backend <topics-feed-storage-ftp>`:
|
||||
|
||||
- It is now possible to set the new ``overwrite`` :ref:`feed option
|
||||
<feed-options>` to ``False`` to append to an existing file instead of
|
||||
overwriting it
|
||||
|
||||
- The FTP password can now be omitted if it is not necessary
|
||||
|
||||
(:issue:`547`, :issue:`716`, :issue:`4512`)
|
||||
|
||||
* The ``__init__`` method of :class:`~scrapy.exporters.CsvItemExporter` now
|
||||
supports an ``errors`` parameter to indicate how to handle encoding errors
|
||||
(:issue:`4755`)
|
||||
|
||||
* When :ref:`using asyncio <using-asyncio>`, it is now possible to
|
||||
:ref:`set a custom asyncio loop <using-custom-loops>` (:issue:`4306`,
|
||||
:issue:`4414`)
|
||||
|
||||
* Serialized requests (see :ref:`topics-jobs`) now support callbacks that are
|
||||
spider methods that delegate on other callable (:issue:`4756`)
|
||||
|
||||
* When a response is larger than :setting:`DOWNLOAD_MAXSIZE`, the logged
|
||||
message is now a warning, instead of an error (:issue:`3874`,
|
||||
:issue:`3886`, :issue:`4752`)
|
||||
|
||||
|
||||
Bug fixes
|
||||
~~~~~~~~~
|
||||
|
||||
* The :command:`genspider` command no longer overwrites existing files
|
||||
unless the ``--force`` option is used (:issue:`4561`, :issue:`4616`,
|
||||
:issue:`4623`)
|
||||
|
||||
* Cookies with an empty value are no longer considered invalid cookies
|
||||
(:issue:`4772`)
|
||||
|
||||
* The :command:`runspider` command now supports files with the ``.pyw`` file
|
||||
extension (:issue:`4643`, :issue:`4646`)
|
||||
|
||||
* The :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
|
||||
middleware now simply ignores unsupported proxy values (:issue:`3331`,
|
||||
:issue:`4778`)
|
||||
|
||||
* Checks for generator callbacks with a ``return`` statement no longer warn
|
||||
about ``return`` statements in nested functions (:issue:`4720`,
|
||||
:issue:`4721`)
|
||||
|
||||
* The system file mode creation mask no longer affects the permissions of
|
||||
files generated using the :command:`startproject` command (:issue:`4722`)
|
||||
|
||||
* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
|
||||
(:issue:`861`, :issue:`4746`)
|
||||
|
||||
* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
|
||||
work when using a headless browser (:issue:`4835`)
|
||||
|
||||
|
||||
Documentation
|
||||
~~~~~~~~~~~~~
|
||||
|
||||
* The :setting:`FEED_URI_PARAMS` setting is now documented (:issue:`4671`,
|
||||
:issue:`4724`)
|
||||
|
||||
* Improved the documentation of
|
||||
:ref:`link extractors <topics-link-extractors>` with an usage example from
|
||||
a spider callback and reference documentation for the
|
||||
:class:`~scrapy.link.Link` class (:issue:`4751`, :issue:`4775`)
|
||||
|
||||
* Clarified the impact of :setting:`CONCURRENT_REQUESTS` when using the
|
||||
:class:`~scrapy.extensions.closespider.CloseSpider` extension
|
||||
(:issue:`4836`)
|
||||
|
||||
* Removed references to Python 2’s ``unicode`` type (:issue:`4547`,
|
||||
:issue:`4703`)
|
||||
|
||||
* We now have an :ref:`official deprecation policy <deprecation-policy>`
|
||||
(:issue:`4705`)
|
||||
|
||||
* Our :ref:`documentation policies <documentation-policies>` now cover usage
|
||||
of Sphinx’s :rst:dir:`versionadded` and :rst:dir:`versionchanged`
|
||||
directives, and we have removed usages referencing Scrapy 1.4.0 and earlier
|
||||
versions (:issue:`3971`, :issue:`4310`)
|
||||
|
||||
* Other documentation cleanups (:issue:`4090`, :issue:`4782`, :issue:`4800`,
|
||||
:issue:`4801`, :issue:`4809`, :issue:`4816`, :issue:`4825`)
|
||||
|
||||
|
||||
Quality assurance
|
||||
~~~~~~~~~~~~~~~~~
|
||||
|
||||
* Extended typing hints (:issue:`4243`, :issue:`4691`)
|
||||
|
||||
* Added tests for the :command:`check` command (:issue:`4663`)
|
||||
|
||||
* Fixed test failures on Debian (:issue:`4726`, :issue:`4727`, :issue:`4735`)
|
||||
|
||||
* Improved Windows test coverage (:issue:`4723`)
|
||||
|
||||
* Switched to :ref:`formatted string literals <f-strings>` where possible
|
||||
(:issue:`4307`, :issue:`4324`, :issue:`4672`)
|
||||
|
||||
* Modernized :func:`super` usage (:issue:`4707`)
|
||||
|
||||
* Other code and test cleanups (:issue:`1790`, :issue:`3288`, :issue:`4165`,
|
||||
:issue:`4564`, :issue:`4651`, :issue:`4714`, :issue:`4738`, :issue:`4745`,
|
||||
:issue:`4747`, :issue:`4761`, :issue:`4765`, :issue:`4804`, :issue:`4817`,
|
||||
:issue:`4820`, :issue:`4822`, :issue:`4839`)
|
||||
|
||||
|
||||
.. _release-2.3.0:
|
||||
|
||||
Scrapy 2.3.0 (2020-08-04)
|
||||
|
|
@ -4008,9 +4335,9 @@ First release of Scrapy.
|
|||
.. _six: https://six.readthedocs.io/
|
||||
.. _tox: https://pypi.org/project/tox/
|
||||
.. _Twisted: https://twistedmatrix.com/trac/
|
||||
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
|
||||
.. _w3lib: https://github.com/scrapy/w3lib
|
||||
.. _w3lib.encoding: https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
|
||||
.. _What is cacheable: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
|
||||
.. _zope.interface: https://zopeinterface.readthedocs.io/en/latest/
|
||||
.. _Zsh: https://www.zsh.org/
|
||||
.. _zstandard: https://pypi.org/project/zstandard/
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
.. _using-asyncio:
|
||||
|
||||
=======
|
||||
asyncio
|
||||
=======
|
||||
|
|
|
|||
|
|
@ -207,6 +207,11 @@ CookiesMiddleware
|
|||
a warning. Refer to :ref:`topics-logging-advanced-customization`
|
||||
to customize the logging behaviour.
|
||||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
The following settings can be used to configure the cookie middleware:
|
||||
|
||||
* :setting:`COOKIES_ENABLED`
|
||||
|
|
@ -684,11 +689,14 @@ HttpCompressionMiddleware
|
|||
This middleware allows compressed (gzip, deflate) traffic to be
|
||||
sent/received from web sites.
|
||||
|
||||
This middleware also supports decoding `brotli-compressed`_ responses,
|
||||
provided `brotlipy`_ is installed.
|
||||
This middleware also supports decoding `brotli-compressed`_ as well as
|
||||
`zstd-compressed`_ responses, provided that `brotlipy`_ or `zstandard`_ is
|
||||
installed, respectively.
|
||||
|
||||
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
||||
.. _brotlipy: https://pypi.org/project/brotlipy/
|
||||
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
|
||||
.. _zstandard: https://pypi.org/project/zstandard/
|
||||
|
||||
HttpCompressionMiddleware Settings
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
|
|
|
|||
|
|
@ -123,7 +123,7 @@ Example::
|
|||
def serialize_field(self, field, name, value):
|
||||
if field == 'price':
|
||||
return f'$ {str(value)}'
|
||||
return super(Product, self).serialize_field(field, name, value)
|
||||
return super().serialize_field(field, name, value)
|
||||
|
||||
.. _topics-exporters-reference:
|
||||
|
||||
|
|
|
|||
|
|
@ -257,6 +257,12 @@ settings:
|
|||
* :setting:`CLOSESPIDER_PAGECOUNT`
|
||||
* :setting:`CLOSESPIDER_ERRORCOUNT`
|
||||
|
||||
.. note::
|
||||
|
||||
When a certain closing condition is met, requests which are
|
||||
currently in the downloader queue (up to :setting:`CONCURRENT_REQUESTS`
|
||||
requests) are still processed.
|
||||
|
||||
.. setting:: CLOSESPIDER_TIMEOUT
|
||||
|
||||
CLOSESPIDER_TIMEOUT
|
||||
|
|
@ -279,8 +285,6 @@ Default: ``0``
|
|||
An integer which specifies a number of items. If the spider scrapes more than
|
||||
that amount and those items are passed by the item pipeline, the
|
||||
spider will be closed with the reason ``closespider_itemcount``.
|
||||
Requests which are currently in the downloader queue (up to
|
||||
:setting:`CONCURRENT_REQUESTS` requests) are still processed.
|
||||
If zero (or non set), spiders won't be closed by number of passed items.
|
||||
|
||||
.. setting:: CLOSESPIDER_PAGECOUNT
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ The feeds are stored on `Amazon S3`_.
|
|||
* ``s3://mybucket/path/to/export.csv``
|
||||
* ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||
|
||||
* Required external libraries: `botocore`_
|
||||
* Required external libraries: `botocore`_ >= 1.4.87
|
||||
|
||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||
passed through the following settings:
|
||||
|
|
@ -303,6 +303,9 @@ For instance::
|
|||
'store_empty': False,
|
||||
'fields': None,
|
||||
'indent': 4,
|
||||
'item_export_kwargs': {
|
||||
'export_empty_fields': True,
|
||||
},
|
||||
},
|
||||
'/home/user/documents/items.xml': {
|
||||
'format': 'xml',
|
||||
|
|
@ -316,6 +319,8 @@ For instance::
|
|||
},
|
||||
}
|
||||
|
||||
.. _feed-options:
|
||||
|
||||
The following is a list of the accepted keys and the setting that is used
|
||||
as a fallback value if that key is not provided for a specific feed definition:
|
||||
|
||||
|
|
@ -326,12 +331,18 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
- ``batch_item_count``: falls back to
|
||||
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
.. versionadded:: 2.3.0
|
||||
|
||||
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
|
||||
|
||||
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class <topics-exporters>`.
|
||||
|
||||
.. versionadded:: 2.4.0
|
||||
|
||||
- ``overwrite``: whether to overwrite the file if it already exists
|
||||
(``True``) or append to its content (``False``).
|
||||
|
||||
|
|
@ -350,6 +361,8 @@ as a fallback value if that key is not provided for a specific feed definition:
|
|||
|
||||
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
|
||||
|
||||
.. versionadded:: 2.4.0
|
||||
|
||||
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
|
||||
|
||||
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
|
||||
|
|
@ -512,7 +525,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
|
|||
.. setting:: FEED_EXPORT_BATCH_ITEM_COUNT
|
||||
|
||||
FEED_EXPORT_BATCH_ITEM_COUNT
|
||||
-----------------------------
|
||||
----------------------------
|
||||
|
||||
.. versionadded:: 2.3.0
|
||||
|
||||
Default: ``0``
|
||||
|
||||
|
|
@ -581,11 +596,15 @@ The function signature should be as follows:
|
|||
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
|
||||
is always ``1``.
|
||||
|
||||
.. versionadded:: 2.3.0
|
||||
|
||||
- ``batch_time``: UTC date and time, in ISO format with ``:``
|
||||
replaced with ``-``.
|
||||
|
||||
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
.. versionadded:: 2.3.0
|
||||
|
||||
- ``time``: ``batch_time``, with microseconds set to ``0``.
|
||||
:type params: dict
|
||||
|
||||
|
|
|
|||
|
|
@ -10,12 +10,19 @@ The ``__init__`` method of
|
|||
:class:`~scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor` takes settings that
|
||||
determine which links may be extracted. :class:`LxmlLinkExtractor.extract_links
|
||||
<scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor.extract_links>` returns a
|
||||
list of matching :class:`scrapy.link.Link` objects from a
|
||||
list of matching :class:`~scrapy.link.Link` objects from a
|
||||
:class:`~scrapy.http.Response` object.
|
||||
|
||||
Link extractors are used in :class:`~scrapy.spiders.CrawlSpider` spiders
|
||||
through a set of :class:`~scrapy.spiders.Rule` objects. You can also use link
|
||||
extractors in regular spiders.
|
||||
through a set of :class:`~scrapy.spiders.Rule` objects.
|
||||
|
||||
You can also use link extractors in regular spiders. For example, you can instantiate
|
||||
:class:`LinkExtractor <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` into a class
|
||||
variable in your spider, and use it from your spider callbacks::
|
||||
|
||||
def parse(self, response):
|
||||
for link in self.link_extractor.extract_links(response):
|
||||
yield Request(link.url, callback=self.parse)
|
||||
|
||||
.. _topics-link-extractors-ref:
|
||||
|
||||
|
|
@ -145,4 +152,12 @@ LxmlLinkExtractor
|
|||
|
||||
.. automethod:: extract_links
|
||||
|
||||
Link
|
||||
----
|
||||
|
||||
.. module:: scrapy.link
|
||||
:synopsis: Link from link extractors
|
||||
|
||||
.. autoclass:: Link
|
||||
|
||||
.. _scrapy.linkextractors: https://github.com/scrapy/scrapy/blob/master/scrapy/linkextractors/__init__.py
|
||||
|
|
|
|||
|
|
@ -56,6 +56,8 @@ this:
|
|||
error will be logged and the file won't be present in the ``files`` field.
|
||||
|
||||
|
||||
.. _images-pipeline:
|
||||
|
||||
Using the Images Pipeline
|
||||
=========================
|
||||
|
||||
|
|
@ -68,14 +70,10 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
|
|||
can configure some extra functions like generating thumbnails and filtering
|
||||
the images based on their size.
|
||||
|
||||
The Images Pipeline uses `Pillow`_ for thumbnailing and normalizing images to
|
||||
JPEG/RGB format, so you need to install this library in order to use it.
|
||||
`Python Imaging Library`_ (PIL) should also work in most cases, but it is known
|
||||
to cause troubles in some setups, so we recommend to use `Pillow`_ instead of
|
||||
PIL.
|
||||
The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
|
||||
thumbnailing and normalizing images to JPEG/RGB format.
|
||||
|
||||
.. _Pillow: https://github.com/python-pillow/Pillow
|
||||
.. _Python Imaging Library: http://www.pythonware.com/products/pil/
|
||||
|
||||
|
||||
.. _topics-media-pipeline-enabling:
|
||||
|
|
@ -164,14 +162,17 @@ FTP supports two different connection modes: active or passive. Scrapy uses
|
|||
the passive connection mode by default. To use the active connection mode instead,
|
||||
set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
|
||||
|
||||
.. _media-pipelines-s3:
|
||||
|
||||
Amazon S3 storage
|
||||
-----------------
|
||||
|
||||
.. setting:: FILES_STORE_S3_ACL
|
||||
.. setting:: IMAGES_STORE_S3_ACL
|
||||
|
||||
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
|
||||
bucket. Scrapy will automatically upload the files to the bucket.
|
||||
If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
|
||||
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
|
||||
automatically upload the files to the bucket.
|
||||
|
||||
For example, this is a valid :setting:`IMAGES_STORE` value::
|
||||
|
||||
|
|
@ -187,8 +188,9 @@ policy::
|
|||
|
||||
For more information, see `canned ACLs`_ in the Amazon S3 Developer Guide.
|
||||
|
||||
Because Scrapy uses ``botocore`` internally you can also use other S3-like storages. Storages like
|
||||
self-hosted `Minio`_ or `s3.scality`_. All you need to do is set endpoint option in you Scrapy settings::
|
||||
You can also use other S3-like storages. Storages like self-hosted `Minio`_ or
|
||||
`s3.scality`_. All you need to do is set endpoint option in you Scrapy
|
||||
settings::
|
||||
|
||||
AWS_ENDPOINT_URL = 'http://minio.example.com:9000'
|
||||
|
||||
|
|
@ -197,9 +199,10 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
|
|||
AWS_USE_SSL = False # or True (None by default)
|
||||
AWS_VERIFY = False # or True (None by default)
|
||||
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
.. _Minio: https://github.com/minio/minio
|
||||
.. _s3.scality: https://s3.scality.com/
|
||||
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
|
||||
|
||||
|
||||
.. _media-pipeline-gcs:
|
||||
|
|
@ -446,6 +449,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
By default the :meth:`file_path` method returns
|
||||
``full/<request URL hash>.<extension>``.
|
||||
|
||||
.. versionadded:: 2.4
|
||||
The *item* parameter.
|
||||
|
||||
.. method:: FilesPipeline.get_media_requests(item, info)
|
||||
|
||||
As seen on the workflow, the pipeline will get the URLs of the images to
|
||||
|
|
@ -582,6 +588,9 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
By default the :meth:`file_path` method returns
|
||||
``full/<request URL hash>.<extension>``.
|
||||
|
||||
.. versionadded:: 2.4
|
||||
The *item* parameter.
|
||||
|
||||
.. method:: ImagesPipeline.get_media_requests(item, info)
|
||||
|
||||
Works the same way as :meth:`FilesPipeline.get_media_requests` method,
|
||||
|
|
|
|||
|
|
@ -61,6 +61,12 @@ Request objects
|
|||
:param headers: the headers of this request. The dict values can be strings
|
||||
(for single valued headers) or lists (for multi-valued headers). If
|
||||
``None`` is passed as value, the HTTP header will not be sent at all.
|
||||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
:type headers: dict
|
||||
|
||||
:param cookies: the request cookies. These can be sent in two forms.
|
||||
|
|
@ -102,6 +108,12 @@ Request objects
|
|||
)
|
||||
|
||||
For more info see :ref:`cookies-mw`.
|
||||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
:type cookies: dict or list
|
||||
|
||||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
|
|
@ -681,9 +693,19 @@ Response objects
|
|||
:param ip_address: The IP address of the server from which the Response originated.
|
||||
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
|
||||
|
||||
:param protocol: The protocol that was used to download the response.
|
||||
For instance: "HTTP/1.0", "HTTP/1.1"
|
||||
:type protocol: :class:`str`
|
||||
|
||||
.. versionadded:: 2.0.0
|
||||
The ``certificate`` parameter.
|
||||
|
||||
.. versionadded:: 2.1.0
|
||||
The ``ip_address`` parameter.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
The ``protocol`` parameter.
|
||||
|
||||
.. attribute:: Response.url
|
||||
|
||||
A string containing the URL of the response.
|
||||
|
|
@ -768,6 +790,8 @@ Response objects
|
|||
|
||||
.. attribute:: Response.certificate
|
||||
|
||||
.. versionadded:: 2.0.0
|
||||
|
||||
A :class:`twisted.internet.ssl.Certificate` object representing
|
||||
the server's SSL certificate.
|
||||
|
||||
|
|
@ -783,6 +807,17 @@ Response objects
|
|||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. attribute:: Response.protocol
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
The protocol that was used to download the response.
|
||||
For instance: "HTTP/1.0", "HTTP/1.1"
|
||||
|
||||
This attribute is currently only populated by the HTTP download
|
||||
handlers, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`protocol` is always ``None``.
|
||||
|
||||
.. method:: Response.copy()
|
||||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ module and documented in the :ref:`topics-settings-ref` section.
|
|||
Import paths and classes
|
||||
========================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
.. versionadded:: 2.4.0
|
||||
|
||||
When a setting references a callable object to be imported by Scrapy, such as a
|
||||
class or a function, there are two different ways you can specify that object:
|
||||
|
|
@ -249,19 +249,25 @@ ASYNCIO_EVENT_LOOP
|
|||
|
||||
Default: ``None``
|
||||
|
||||
Import path of a given asyncio event loop class.
|
||||
Import path of a given ``asyncio`` event loop class.
|
||||
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||
event loop will be used.
|
||||
|
||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
|
||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||
|
||||
.. caution:: Please be aware that, when using a non-default event loop
|
||||
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
||||
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
||||
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
||||
as the current loop for the current OS thread.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
BOT_NAME
|
||||
|
|
@ -352,6 +358,11 @@ Default::
|
|||
The default headers used for Scrapy HTTP Requests. They're populated in the
|
||||
:class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
|
||||
|
||||
.. caution:: Cookies set via the ``Cookie`` header are not considered by the
|
||||
:ref:`cookies-mw`. If you need to set cookies for a request, use the
|
||||
:class:`Request.cookies <scrapy.http.Request>` parameter. This is a known
|
||||
current limitation that is being worked on.
|
||||
|
||||
.. setting:: DEPTH_LIMIT
|
||||
|
||||
DEPTH_LIMIT
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
[pytest]
|
||||
xfail_strict = true
|
||||
usefixtures = chdir
|
||||
python_files=test_*.py __init__.py
|
||||
python_classes=
|
||||
|
|
@ -35,9 +36,5 @@ flake8-ignore =
|
|||
scrapy/spiders/__init__.py E402 F401
|
||||
|
||||
# Issues pending a review:
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/markup.py F403
|
||||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/url.py F403 F405
|
||||
tests/test_loader.py E741
|
||||
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
2.3.0
|
||||
2.4.1
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ class Command(ScrapyCommand):
|
|||
print(f"Created spider {name!r} using template {template_name!r} ",
|
||||
end=('' if spiders_module else '\n'))
|
||||
if spiders_module:
|
||||
print("in module:\n {spiders_module.__name__}.{module}")
|
||||
print(f"in module:\n {spiders_module.__name__}.{module}")
|
||||
|
||||
def _find_template(self, template):
|
||||
template_file = join(self.templates_dir, f'{template}.tmpl')
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ def _import_file(filepath):
|
|||
abspath = os.path.abspath(filepath)
|
||||
dirname, file = os.path.split(abspath)
|
||||
fname, fext = os.path.splitext(file)
|
||||
if fext != '.py':
|
||||
if fext not in ('.py', '.pyw'):
|
||||
raise ValueError(f"Not a Python source file: {abspath}")
|
||||
if dirname:
|
||||
sys.path = [dirname] + sys.path
|
||||
|
|
|
|||
|
|
@ -434,6 +434,11 @@ class ScrapyAgent:
|
|||
def _cb_bodydone(self, result, request, url):
|
||||
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
|
||||
try:
|
||||
version = result["txresponse"].version
|
||||
protocol = f"{to_unicode(version[0])}/{version[1]}.{version[2]}"
|
||||
except (AttributeError, TypeError, IndexError):
|
||||
protocol = None
|
||||
response = respcls(
|
||||
url=url,
|
||||
status=int(result["txresponse"].code),
|
||||
|
|
@ -442,6 +447,7 @@ class ScrapyAgent:
|
|||
flags=result["flags"],
|
||||
certificate=result["certificate"],
|
||||
ip_address=result["ip_address"],
|
||||
protocol=protocol,
|
||||
)
|
||||
if result.get("failure"):
|
||||
result["failure"].value.response = response
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from service_identity.exceptions import CertificateError
|
|||
from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError
|
||||
from twisted.internet.ssl import AcceptableCiphers
|
||||
|
||||
from scrapy import twisted_version
|
||||
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
|
||||
|
||||
|
||||
|
|
@ -28,13 +27,6 @@ openssl_methods = {
|
|||
}
|
||||
|
||||
|
||||
if twisted_version < (17, 0, 0):
|
||||
from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name
|
||||
else:
|
||||
def set_tlsext_host_name(connection, hostNameBytes):
|
||||
connection.set_tlsext_host_name(hostNameBytes)
|
||||
|
||||
|
||||
class ScrapyClientTLSOptions(ClientTLSOptions):
|
||||
"""
|
||||
SSL Client connection creator ignoring certificate verification errors
|
||||
|
|
@ -52,21 +44,14 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
|
||||
def _identityVerifyingInfoCallback(self, connection, where, ret):
|
||||
if where & SSL.SSL_CB_HANDSHAKE_START:
|
||||
set_tlsext_host_name(connection, self._hostnameBytes)
|
||||
connection.set_tlsext_host_name(self._hostnameBytes)
|
||||
elif where & SSL.SSL_CB_HANDSHAKE_DONE:
|
||||
if self.verbose_logging:
|
||||
if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
|
||||
if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
|
||||
logger.debug('SSL connection to %s using protocol %s, cipher %s',
|
||||
self._hostnameASCII,
|
||||
connection.get_protocol_version_name(),
|
||||
connection.get_cipher_name(),
|
||||
)
|
||||
else:
|
||||
logger.debug('SSL connection to %s using cipher %s',
|
||||
self._hostnameASCII,
|
||||
connection.get_cipher_name(),
|
||||
)
|
||||
logger.debug('SSL connection to %s using protocol %s, cipher %s',
|
||||
self._hostnameASCII,
|
||||
connection.get_protocol_version_name(),
|
||||
connection.get_cipher_name(),
|
||||
)
|
||||
server_cert = connection.get_peer_certificate()
|
||||
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
|
||||
x509name_to_string(server_cert.get_issuer()),
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from twisted.internet.protocol import ClientFactory
|
|||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.python import to_bytes
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.responsetypes import responsetypes
|
||||
|
||||
|
||||
|
|
@ -110,7 +110,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
|
|||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
parsed = urlparse_cached(request)
|
||||
|
|
|
|||
|
|
@ -1,14 +1,10 @@
|
|||
import os
|
||||
import json
|
||||
import logging
|
||||
import warnings
|
||||
from os.path import join, exists
|
||||
|
||||
from queuelib import PriorityQueue
|
||||
|
||||
from scrapy.utils.misc import load_object, create_instance
|
||||
from scrapy.utils.job import job_dir
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -56,14 +52,6 @@ class Scheduler:
|
|||
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
|
||||
dupefilter = create_instance(dupefilter_cls, settings, crawler)
|
||||
pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
|
||||
if pqclass is PriorityQueue:
|
||||
warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'"
|
||||
" is no longer supported because of API changes; "
|
||||
"please use 'scrapy.pqueues.ScrapyPriorityQueue'",
|
||||
ScrapyDeprecationWarning)
|
||||
from scrapy.pqueues import ScrapyPriorityQueue
|
||||
pqclass = ScrapyPriorityQueue
|
||||
|
||||
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
|
||||
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
|
||||
logunser = settings.getbool('SCHEDULER_DEBUG')
|
||||
|
|
|
|||
|
|
@ -180,9 +180,9 @@ class CrawlerRunner:
|
|||
:type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance,
|
||||
:class:`~scrapy.spiders.Spider` subclass or string
|
||||
|
||||
:param list args: arguments to initialize the spider
|
||||
:param args: arguments to initialize the spider
|
||||
|
||||
:param dict kwargs: keyword arguments to initialize the spider
|
||||
:param kwargs: keyword arguments to initialize the spider
|
||||
"""
|
||||
if isinstance(crawler_or_spidercls, Spider):
|
||||
raise ValueError(
|
||||
|
|
|
|||
|
|
@ -97,35 +97,14 @@ class CookiesMiddleware:
|
|||
|
||||
def _get_request_cookies(self, jar, request):
|
||||
"""
|
||||
Extract cookies from a Request. Values from the `Request.cookies` attribute
|
||||
take precedence over values from the `Cookie` request header.
|
||||
Extract cookies from the Request.cookies attribute
|
||||
"""
|
||||
def get_cookies_from_header(jar, request):
|
||||
cookie_header = request.headers.get("Cookie")
|
||||
if not cookie_header:
|
||||
return []
|
||||
cookie_gen_bytes = (s.strip() for s in cookie_header.split(b";"))
|
||||
cookie_list_unicode = []
|
||||
for cookie_bytes in cookie_gen_bytes:
|
||||
try:
|
||||
cookie_unicode = cookie_bytes.decode("utf8")
|
||||
except UnicodeDecodeError:
|
||||
logger.warning("Non UTF-8 encoded cookie found in request %s: %s",
|
||||
request, cookie_bytes)
|
||||
cookie_unicode = cookie_bytes.decode("latin1", errors="replace")
|
||||
cookie_list_unicode.append(cookie_unicode)
|
||||
response = Response(request.url, headers={"Set-Cookie": cookie_list_unicode})
|
||||
return jar.make_cookies(response, request)
|
||||
|
||||
def get_cookies_from_attribute(jar, request):
|
||||
if not request.cookies:
|
||||
return []
|
||||
elif isinstance(request.cookies, dict):
|
||||
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
cookies = request.cookies
|
||||
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
|
||||
response = Response(request.url, headers={"Set-Cookie": formatted})
|
||||
return jar.make_cookies(response, request)
|
||||
|
||||
return get_cookies_from_header(jar, request) + get_cookies_from_attribute(jar, request)
|
||||
if not request.cookies:
|
||||
return []
|
||||
elif isinstance(request.cookies, dict):
|
||||
cookies = ({"name": k, "value": v} for k, v in request.cookies.items())
|
||||
else:
|
||||
cookies = request.cookies
|
||||
formatted = filter(None, (self._format_cookie(c, request) for c in cookies))
|
||||
response = Response(request.url, headers={"Set-Cookie": formatted})
|
||||
return jar.make_cookies(response, request)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import io
|
||||
import zlib
|
||||
|
||||
from scrapy.utils.gz import gunzip
|
||||
|
|
@ -14,6 +15,12 @@ try:
|
|||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
import zstandard
|
||||
ACCEPTED_ENCODINGS.append(b'zstd')
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
|
||||
class HttpCompressionMiddleware:
|
||||
"""This middleware allows compressed (gzip, deflate) traffic to be
|
||||
|
|
@ -67,4 +74,9 @@ class HttpCompressionMiddleware:
|
|||
body = zlib.decompress(body, -15)
|
||||
if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
|
||||
body = brotli.decompress(body)
|
||||
if encoding == b'zstd' and b'zstd' in ACCEPTED_ENCODINGS:
|
||||
# Using its streaming API since its simple API could handle only cases
|
||||
# where there is content size data embedded in the frame
|
||||
reader = zstandard.ZstdDecompressor().stream_reader(io.BytesIO(body))
|
||||
body = reader.read()
|
||||
return body
|
||||
|
|
|
|||
|
|
@ -13,7 +13,12 @@ class HttpProxyMiddleware:
|
|||
self.auth_encoding = auth_encoding
|
||||
self.proxies = {}
|
||||
for type_, url in getproxies().items():
|
||||
self.proxies[type_] = self._get_proxy(url, type_)
|
||||
try:
|
||||
self.proxies[type_] = self._get_proxy(url, type_)
|
||||
# some values such as '/var/run/docker.sock' can't be parsed
|
||||
# by _parse_proxy and as such should be skipped
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
|
|
|
|||
|
|
@ -319,18 +319,26 @@ class FeedExporter:
|
|||
# Use `largs=log_args` to copy log_args into function's scope
|
||||
# instead of using `log_args` from the outer scope
|
||||
d.addCallback(
|
||||
lambda _, largs=log_args: logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
)
|
||||
self._handle_store_success, log_args, logfmt, spider, type(slot.storage).__name__
|
||||
)
|
||||
d.addErrback(
|
||||
lambda f, largs=log_args: logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
self._handle_store_error, log_args, logfmt, spider, type(slot.storage).__name__
|
||||
)
|
||||
return d
|
||||
|
||||
def _handle_store_error(self, f, largs, logfmt, spider, slot_type):
|
||||
logger.error(
|
||||
logfmt % "Error storing", largs,
|
||||
exc_info=failure_to_exc_info(f), extra={'spider': spider}
|
||||
)
|
||||
self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
|
||||
|
||||
def _handle_store_success(self, f, largs, logfmt, spider, slot_type):
|
||||
logger.info(
|
||||
logfmt % "Stored", largs, extra={'spider': spider}
|
||||
)
|
||||
self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
|
||||
|
||||
def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
|
||||
"""
|
||||
Redirect the output data stream to a new file.
|
||||
|
|
@ -349,6 +357,7 @@ class FeedExporter:
|
|||
fields_to_export=feed_options['fields'],
|
||||
encoding=feed_options['encoding'],
|
||||
indent=feed_options['indent'],
|
||||
**feed_options['item_export_kwargs'],
|
||||
)
|
||||
slot = _FeedSlot(
|
||||
file=file,
|
||||
|
|
@ -451,7 +460,7 @@ class FeedExporter:
|
|||
crawler = getattr(self, 'crawler', None)
|
||||
|
||||
def build_instance(builder, *preargs):
|
||||
return build_storage(builder, uri, preargs=preargs)
|
||||
return build_storage(builder, uri, feed_options=feed_options, preargs=preargs)
|
||||
|
||||
if crawler and hasattr(feedcls, 'from_crawler'):
|
||||
instance = build_instance(feedcls.from_crawler, crawler)
|
||||
|
|
|
|||
|
|
@ -65,7 +65,11 @@ class Request(object_ref):
|
|||
s = safe_url_string(url, self.encoding)
|
||||
self._url = escape_ajax(s)
|
||||
|
||||
if ('://' not in self._url) and (not self._url.startswith('data:')):
|
||||
if (
|
||||
'://' not in self._url
|
||||
and not self._url.startswith('about:')
|
||||
and not self._url.startswith('data:')
|
||||
):
|
||||
raise ValueError(f'Missing scheme in request url: {self._url}')
|
||||
|
||||
url = property(_get_url, obsolete_setter(_set_url, 'url'))
|
||||
|
|
|
|||
|
|
@ -17,8 +17,18 @@ from scrapy.utils.trackref import object_ref
|
|||
|
||||
class Response(object_ref):
|
||||
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None,
|
||||
request=None, certificate=None, ip_address=None):
|
||||
def __init__(
|
||||
self,
|
||||
url,
|
||||
status=200,
|
||||
headers=None,
|
||||
body=b"",
|
||||
flags=None,
|
||||
request=None,
|
||||
certificate=None,
|
||||
ip_address=None,
|
||||
protocol=None,
|
||||
):
|
||||
self.headers = Headers(headers or {})
|
||||
self.status = int(status)
|
||||
self._set_body(body)
|
||||
|
|
@ -27,6 +37,7 @@ class Response(object_ref):
|
|||
self.flags = [] if flags is None else list(flags)
|
||||
self.certificate = certificate
|
||||
self.ip_address = ip_address
|
||||
self.protocol = protocol
|
||||
|
||||
@property
|
||||
def cb_kwargs(self):
|
||||
|
|
@ -89,8 +100,9 @@ class Response(object_ref):
|
|||
"""Create a new Response with the same attributes except for those
|
||||
given new values.
|
||||
"""
|
||||
for x in ['url', 'status', 'headers', 'body',
|
||||
'request', 'flags', 'certificate', 'ip_address']:
|
||||
for x in [
|
||||
"url", "status", "headers", "body", "request", "flags", "certificate", "ip_address", "protocol",
|
||||
]:
|
||||
kwargs.setdefault(x, getattr(self, x))
|
||||
cls = kwargs.pop('cls', self.__class__)
|
||||
return cls(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,22 @@ its documentation in: docs/topics/link-extractors.rst
|
|||
|
||||
|
||||
class Link:
|
||||
"""Link objects represent an extracted link by the LinkExtractor."""
|
||||
"""Link objects represent an extracted link by the LinkExtractor.
|
||||
|
||||
Using the anchor tag sample below to illustrate the parameters::
|
||||
|
||||
<a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
|
||||
|
||||
:param url: the absolute url being linked to in the anchor tag.
|
||||
From the sample, this is ``https://example.com/nofollow.html``.
|
||||
|
||||
:param text: the text in the anchor tag. From the sample, this is ``Dont follow this one``.
|
||||
|
||||
:param fragment: the part of the url after the hash symbol. From the sample, this is ``foo``.
|
||||
|
||||
:param nofollow: an indication of the presence or absence of a nofollow value in the ``rel`` attribute
|
||||
of the anchor tag.
|
||||
"""
|
||||
|
||||
__slots__ = ['url', 'text', 'fragment', 'nofollow']
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from twisted.internet import defer
|
||||
from twisted.internet.base import ThreadedResolver
|
||||
from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
|
||||
from twisted.internet.interfaces import IHostResolution, IHostnameResolver, IResolutionReceiver, IResolverSimple
|
||||
from zope.interface.declarations import implementer, provider
|
||||
|
||||
from scrapy.utils.datatypes import LocalCache
|
||||
|
|
@ -50,6 +50,36 @@ class CachingThreadedResolver(ThreadedResolver):
|
|||
return result
|
||||
|
||||
|
||||
@implementer(IHostResolution)
|
||||
class HostResolution:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
def cancel(self):
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@provider(IResolutionReceiver)
|
||||
class _CachingResolutionReceiver:
|
||||
def __init__(self, resolutionReceiver, hostName):
|
||||
self.resolutionReceiver = resolutionReceiver
|
||||
self.hostName = hostName
|
||||
self.addresses = []
|
||||
|
||||
def resolutionBegan(self, resolution):
|
||||
self.resolutionReceiver.resolutionBegan(resolution)
|
||||
self.resolution = resolution
|
||||
|
||||
def addressResolved(self, address):
|
||||
self.resolutionReceiver.addressResolved(address)
|
||||
self.addresses.append(address)
|
||||
|
||||
def resolutionComplete(self):
|
||||
self.resolutionReceiver.resolutionComplete()
|
||||
if self.addresses:
|
||||
dnscache[self.hostName] = self.addresses
|
||||
|
||||
|
||||
@implementer(IHostnameResolver)
|
||||
class CachingHostnameResolver:
|
||||
"""
|
||||
|
|
@ -73,33 +103,22 @@ class CachingHostnameResolver:
|
|||
def install_on_reactor(self):
|
||||
self.reactor.installNameResolver(self)
|
||||
|
||||
def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
|
||||
addressTypes=None, transportSemantics='TCP'):
|
||||
|
||||
@provider(IResolutionReceiver)
|
||||
class CachingResolutionReceiver(resolutionReceiver):
|
||||
|
||||
def resolutionBegan(self, resolution):
|
||||
super().resolutionBegan(resolution)
|
||||
self.resolution = resolution
|
||||
self.resolved = False
|
||||
|
||||
def addressResolved(self, address):
|
||||
super().addressResolved(address)
|
||||
self.resolved = True
|
||||
|
||||
def resolutionComplete(self):
|
||||
super().resolutionComplete()
|
||||
if self.resolved:
|
||||
dnscache[hostName] = self.resolution
|
||||
|
||||
def resolveHostName(
|
||||
self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics="TCP"
|
||||
):
|
||||
try:
|
||||
return dnscache[hostName]
|
||||
addresses = dnscache[hostName]
|
||||
except KeyError:
|
||||
return self.original_resolver.resolveHostName(
|
||||
CachingResolutionReceiver(),
|
||||
_CachingResolutionReceiver(resolutionReceiver, hostName),
|
||||
hostName,
|
||||
portNumber,
|
||||
addressTypes,
|
||||
transportSemantics
|
||||
transportSemantics,
|
||||
)
|
||||
else:
|
||||
resolutionReceiver.resolutionBegan(HostResolution(hostName))
|
||||
for addr in addresses:
|
||||
resolutionReceiver.addressResolved(addr)
|
||||
resolutionReceiver.resolutionComplete()
|
||||
return resolutionReceiver
|
||||
|
|
|
|||
|
|
@ -6,14 +6,11 @@ See documentation in docs/topics/spiders.rst
|
|||
"""
|
||||
|
||||
import copy
|
||||
import warnings
|
||||
from typing import Sequence
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import Request, HtmlResponse
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.python import get_func_args
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
||||
|
||||
|
|
@ -37,15 +34,22 @@ _default_link_extractor = LinkExtractor()
|
|||
|
||||
class Rule:
|
||||
|
||||
def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
|
||||
process_links=None, process_request=None, errback=None):
|
||||
def __init__(
|
||||
self,
|
||||
link_extractor=None,
|
||||
callback=None,
|
||||
cb_kwargs=None,
|
||||
follow=None,
|
||||
process_links=None,
|
||||
process_request=None,
|
||||
errback=None,
|
||||
):
|
||||
self.link_extractor = link_extractor or _default_link_extractor
|
||||
self.callback = callback
|
||||
self.errback = errback
|
||||
self.cb_kwargs = cb_kwargs or {}
|
||||
self.process_links = process_links or _identity
|
||||
self.process_request = process_request or _identity_process_request
|
||||
self.process_request_argcount = None
|
||||
self.follow = follow if follow is not None else not callback
|
||||
|
||||
def _compile(self, spider):
|
||||
|
|
@ -53,22 +57,6 @@ class Rule:
|
|||
self.errback = _get_method(self.errback, spider)
|
||||
self.process_links = _get_method(self.process_links, spider)
|
||||
self.process_request = _get_method(self.process_request, spider)
|
||||
self.process_request_argcount = len(get_func_args(self.process_request))
|
||||
if self.process_request_argcount == 1:
|
||||
warnings.warn(
|
||||
"Rule.process_request should accept two arguments "
|
||||
"(request, response), accepting only one is deprecated",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
||||
def _process_request(self, request, response):
|
||||
"""
|
||||
Wrapper around the request processing function to maintain backward
|
||||
compatibility with functions that do not take a Response object
|
||||
"""
|
||||
args = [request] if self.process_request_argcount == 1 else [request, response]
|
||||
return self.process_request(*args)
|
||||
|
||||
|
||||
class CrawlSpider(Spider):
|
||||
|
|
@ -111,7 +99,7 @@ class CrawlSpider(Spider):
|
|||
for link in rule.process_links(links):
|
||||
seen.add(link)
|
||||
request = self._build_request(rule_index, link)
|
||||
yield rule._process_request(request, response)
|
||||
yield rule.process_request(request, response)
|
||||
|
||||
def _callback(self, response):
|
||||
rule = self._rules[response.meta['rule']]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,5 @@
|
|||
async def collect_asyncgen(result):
|
||||
results = []
|
||||
async for x in result:
|
||||
results.append(x)
|
||||
return results
|
||||
|
|
@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
|
|||
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
|
||||
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
|
||||
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
|
||||
out.setdefault("item_export_kwargs", dict())
|
||||
if settings["FEED_EXPORT_INDENT"] is None:
|
||||
out.setdefault("indent", None)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import struct
|
||||
from gzip import GzipFile
|
||||
from io import BytesIO
|
||||
import re
|
||||
import struct
|
||||
|
||||
from scrapy.utils.decorators import deprecated
|
||||
|
||||
|
|
@ -42,17 +41,5 @@ def gunzip(data):
|
|||
return b''.join(output_list)
|
||||
|
||||
|
||||
_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
|
||||
_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
|
||||
|
||||
|
||||
@deprecated
|
||||
def is_gzipped(response):
|
||||
"""Return True if the response is gzipped, or False otherwise"""
|
||||
ctype = response.headers.get('Content-Type', b'')
|
||||
cenc = response.headers.get('Content-Encoding', b'').lower()
|
||||
return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
|
||||
|
||||
|
||||
def gzip_magic_number(response):
|
||||
return response.body[:3] == b'\x1f\x8b\x08'
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
"""
|
||||
Transitional module for moving to the w3lib library.
|
||||
|
||||
For new code, always import from w3lib.http instead of this module
|
||||
"""
|
||||
|
||||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.decorators import deprecated
|
||||
from w3lib.http import * # noqa: F401
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.utils.http` is deprecated, "
|
||||
"Please import from `w3lib.http` instead.",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
||||
|
||||
@deprecated
|
||||
def decode_chunked_transfer(chunked_body):
|
||||
"""Parsed body received with chunked transfer encoding, and return the
|
||||
decoded body.
|
||||
|
||||
For more info see:
|
||||
https://en.wikipedia.org/wiki/Chunked_transfer_encoding
|
||||
|
||||
"""
|
||||
body, h, t = '', '', chunked_body
|
||||
while t:
|
||||
h, t = t.split('\r\n', 1)
|
||||
if h == '0':
|
||||
break
|
||||
size = int(h, 16)
|
||||
body += t[:size]
|
||||
t = t[size + 2:]
|
||||
return body
|
||||
|
|
@ -22,25 +22,41 @@ def xmliter(obj, nodename):
|
|||
"""
|
||||
nodename_patt = re.escape(nodename)
|
||||
|
||||
HEADER_START_RE = re.compile(fr'^(.*?)<\s*{nodename_patt}(?:\s|>)', re.S)
|
||||
DOCUMENT_HEADER_RE = re.compile(r'<\?xml[^>]+>\s*', re.S)
|
||||
HEADER_END_RE = re.compile(fr'<\s*/{nodename_patt}\s*>', re.S)
|
||||
END_TAG_RE = re.compile(r'<\s*/([^\s>]+)\s*>', re.S)
|
||||
NAMESPACE_RE = re.compile(r'((xmlns[:A-Za-z]*)=[^>\s]+)', re.S)
|
||||
text = _body_or_str(obj)
|
||||
|
||||
header_start = re.search(HEADER_START_RE, text)
|
||||
header_start = header_start.group(1).strip() if header_start else ''
|
||||
header_end = re_rsearch(HEADER_END_RE, text)
|
||||
header_end = text[header_end[1]:].strip() if header_end else ''
|
||||
document_header = re.search(DOCUMENT_HEADER_RE, text)
|
||||
document_header = document_header.group().strip() if document_header else ''
|
||||
header_end_idx = re_rsearch(HEADER_END_RE, text)
|
||||
header_end = text[header_end_idx[1]:].strip() if header_end_idx else ''
|
||||
namespaces = {}
|
||||
if header_end:
|
||||
for tagname in reversed(re.findall(END_TAG_RE, header_end)):
|
||||
tag = re.search(fr'<\s*{tagname}.*?xmlns[:=][^>]*>', text[:header_end_idx[1]], re.S)
|
||||
if tag:
|
||||
namespaces.update(reversed(x) for x in re.findall(NAMESPACE_RE, tag.group()))
|
||||
|
||||
r = re.compile(fr'<{nodename_patt}[\s>].*?</{nodename_patt}>', re.DOTALL)
|
||||
for match in r.finditer(text):
|
||||
nodetext = header_start + match.group() + header_end
|
||||
yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
|
||||
nodetext = (
|
||||
document_header
|
||||
+ match.group().replace(
|
||||
nodename,
|
||||
f'{nodename} {" ".join(namespaces.values())}',
|
||||
1
|
||||
)
|
||||
+ header_end
|
||||
)
|
||||
yield Selector(text=nodetext, type='xml')
|
||||
|
||||
|
||||
def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
|
||||
from lxml import etree
|
||||
reader = _StreamReader(obj)
|
||||
tag = f'{{{namespace}}}{nodename}'if namespace else nodename
|
||||
tag = f'{{{namespace}}}{nodename}' if namespace else nodename
|
||||
iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
|
||||
selxpath = '//' + (f'{prefix}:{nodename}' if namespace else nodename)
|
||||
for _, node in iterable:
|
||||
|
|
|
|||
|
|
@ -1,14 +0,0 @@
|
|||
"""
|
||||
Transitional module for moving to the w3lib library.
|
||||
|
||||
For new code, always import from w3lib.html instead of this module
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from w3lib.html import * # noqa: F401
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.utils.markup` is deprecated. "
|
||||
"Please import from `w3lib.html` instead.",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
"""
|
||||
Transitional module for moving to the w3lib library.
|
||||
|
||||
For new code, always import from w3lib.form instead of this module
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from w3lib.form import * # noqa: F401
|
||||
|
||||
|
||||
warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
|
||||
"If you're using `encode_multipart` function, please use "
|
||||
"`urllib3.filepost.encode_multipart_formdata` instead",
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import pickle
|
||||
import warnings
|
||||
|
||||
from importlib import import_module
|
||||
|
|
@ -68,18 +67,10 @@ def get_project_settings():
|
|||
if settings_module_path:
|
||||
settings.setmodule(settings_module_path, priority='project')
|
||||
|
||||
pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
|
||||
if pickled_settings:
|
||||
warnings.warn("Use of environment variable "
|
||||
"'SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE' "
|
||||
"is deprecated.", ScrapyDeprecationWarning)
|
||||
settings.setdict(pickle.loads(pickled_settings), priority='project')
|
||||
|
||||
scrapy_envvars = {k[7:]: v for k, v in os.environ.items() if
|
||||
k.startswith('SCRAPY_')}
|
||||
valid_envvars = {
|
||||
'CHECK',
|
||||
'PICKLED_SETTINGS_TO_OVERRIDE',
|
||||
'PROJECT',
|
||||
'PYTHON_SHELL',
|
||||
'SETTINGS_MODULE',
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
"""
|
||||
Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
|
||||
"""
|
||||
import warnings
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.asyncgen import collect_asyncgen # noqa: F401
|
||||
|
||||
|
||||
async def collect_asyncgen(result):
|
||||
results = []
|
||||
async for x in result:
|
||||
results.append(x)
|
||||
return results
|
||||
warnings.warn(
|
||||
"Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -60,8 +60,9 @@ def install_reactor(reactor_path, event_loop_path=None):
|
|||
if event_loop_path is not None:
|
||||
event_loop_class = load_object(event_loop_path)
|
||||
event_loop = event_loop_class()
|
||||
asyncio.set_event_loop(event_loop)
|
||||
else:
|
||||
event_loop = asyncio.new_event_loop()
|
||||
event_loop = asyncio.get_event_loop()
|
||||
asyncioreactor.install(eventloop=event_loop)
|
||||
else:
|
||||
*module, _ = reactor_path.split(".")
|
||||
|
|
|
|||
|
|
@ -4,17 +4,14 @@ import logging
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.defer import deferred_from_coro
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
try:
|
||||
from scrapy.utils.py36 import collect_asyncgen
|
||||
except SyntaxError:
|
||||
collect_asyncgen = None
|
||||
from scrapy.utils.asyncgen import collect_asyncgen
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def iterate_spider_output(result):
|
||||
if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
|
||||
if inspect.isasyncgen(result):
|
||||
d = deferred_from_coro(collect_asyncgen(result))
|
||||
d.addCallback(iterate_spider_output)
|
||||
return d
|
||||
|
|
|
|||
|
|
@ -13,15 +13,6 @@ from twisted.trial.unittest import SkipTest
|
|||
from scrapy.utils.boto import is_botocore_available
|
||||
|
||||
|
||||
def assert_aws_environ():
|
||||
"""Asserts the current environment is suitable for running AWS testsi.
|
||||
Raises SkipTest with the reason if it's not.
|
||||
"""
|
||||
skip_if_no_boto()
|
||||
if 'AWS_ACCESS_KEY_ID' not in os.environ:
|
||||
raise SkipTest("AWS keys not found")
|
||||
|
||||
|
||||
def assert_gcs_environ():
|
||||
if 'GCS_PROJECT_ID' not in os.environ:
|
||||
raise SkipTest("GCS_PROJECT_ID not found")
|
||||
|
|
@ -32,18 +23,6 @@ def skip_if_no_boto():
|
|||
raise SkipTest('missing botocore library')
|
||||
|
||||
|
||||
def get_s3_content_and_delete(bucket, path, with_key=False):
|
||||
""" Get content from s3 key, and delete key afterwards.
|
||||
"""
|
||||
import botocore.session
|
||||
session = botocore.session.get_session()
|
||||
client = session.create_client('s3')
|
||||
key = client.get_object(Bucket=bucket, Key=path)
|
||||
content = key['Body'].read()
|
||||
client.delete_object(Bucket=bucket, Key=path)
|
||||
return (content, key) if with_key else content
|
||||
|
||||
|
||||
def get_gcs_content_and_delete(bucket, path):
|
||||
from google.cloud import storage
|
||||
client = storage.Client(project=os.environ.get('GCS_PROJECT_ID'))
|
||||
|
|
|
|||
|
|
@ -55,9 +55,6 @@ ignore_errors = True
|
|||
[mypy-scrapy.utils.response]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.spider]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-scrapy.utils.trackref]
|
||||
ignore_errors = True
|
||||
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -56,7 +56,7 @@ setup(
|
|||
name='Scrapy',
|
||||
version=version,
|
||||
url='https://scrapy.org',
|
||||
project_urls = {
|
||||
project_urls={
|
||||
'Documentation': 'https://docs.scrapy.org/',
|
||||
'Source': 'https://github.com/scrapy/scrapy',
|
||||
'Tracker': 'https://github.com/scrapy/scrapy/issues',
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class IPv6Spider(scrapy.Spider):
|
||||
name = "ipv6_spider"
|
||||
start_urls = ["http://[::1]"]
|
||||
|
||||
|
||||
process = CrawlerProcess(settings={
|
||||
"RETRY_ENABLED": False,
|
||||
"DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
|
||||
})
|
||||
process.crawl(IPv6Spider)
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
import asyncio
|
||||
import sys
|
||||
|
||||
from scrapy import Spider
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
from scrapy.utils.defer import deferred_from_coro
|
||||
from twisted.internet.defer import Deferred
|
||||
|
||||
|
||||
class UppercasePipeline:
|
||||
async def _open_spider(self, spider):
|
||||
spider.logger.info("async pipeline opened!")
|
||||
await asyncio.sleep(0.1)
|
||||
|
||||
def open_spider(self, spider):
|
||||
return deferred_from_coro(self._open_spider(spider))
|
||||
|
||||
def process_item(self, item, spider):
|
||||
return {"url": item["url"].upper()}
|
||||
|
||||
|
||||
class UrlSpider(Spider):
|
||||
name = "url_spider"
|
||||
start_urls = ["data:,"]
|
||||
custom_settings = {
|
||||
"ITEM_PIPELINES": {UppercasePipeline: 100},
|
||||
}
|
||||
|
||||
def parse(self, response):
|
||||
yield {"url": response.url}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
ASYNCIO_EVENT_LOOP = sys.argv[1]
|
||||
except IndexError:
|
||||
ASYNCIO_EVENT_LOOP = None
|
||||
|
||||
process = CrawlerProcess(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP,
|
||||
})
|
||||
process.crawl(UrlSpider)
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
import sys
|
||||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class CachingHostnameResolverSpider(scrapy.Spider):
|
||||
"""
|
||||
Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution)
|
||||
"""
|
||||
name = "caching_hostname_resolver_spider"
|
||||
|
||||
def start_requests(self):
|
||||
yield scrapy.Request(self.url)
|
||||
|
||||
def parse(self, response):
|
||||
for _ in range(10):
|
||||
yield scrapy.Request(response.url, dont_filter=True, callback=self.ignore_response)
|
||||
|
||||
def ignore_response(self, response):
|
||||
self.logger.info(repr(response.ip_address))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
process = CrawlerProcess(settings={
|
||||
"RETRY_ENABLED": False,
|
||||
"DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
|
||||
})
|
||||
process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class CachingHostnameResolverSpider(scrapy.Spider):
|
||||
"""
|
||||
Finishes without a twisted.internet.error.DNSLookupError exception
|
||||
"""
|
||||
name = "caching_hostname_resolver_spider"
|
||||
start_urls = ["http://[::1]"]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
process = CrawlerProcess(settings={
|
||||
"RETRY_ENABLED": False,
|
||||
"DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
|
||||
})
|
||||
process.crawl(CachingHostnameResolverSpider)
|
||||
process.start()
|
||||
|
|
@ -3,10 +3,15 @@ from scrapy.crawler import CrawlerProcess
|
|||
|
||||
|
||||
class IPv6Spider(scrapy.Spider):
|
||||
"""
|
||||
Raises a twisted.internet.error.DNSLookupError:
|
||||
the default name resolver does not handle IPv6 addresses.
|
||||
"""
|
||||
name = "ipv6_spider"
|
||||
start_urls = ["http://[::1]"]
|
||||
|
||||
|
||||
process = CrawlerProcess(settings={"RETRY_ENABLED": False})
|
||||
process.crawl(IPv6Spider)
|
||||
process.start()
|
||||
if __name__ == "__main__":
|
||||
process = CrawlerProcess(settings={"RETRY_ENABLED": False})
|
||||
process.crawl(IPv6Spider)
|
||||
process.start()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ dataclasses; python_version == '3.6'
|
|||
pyftpdlib
|
||||
# https://github.com/pytest-dev/pytest-twisted/issues/93
|
||||
pytest != 5.4, != 5.4.1
|
||||
pytest-azurepipelines
|
||||
pytest-cov
|
||||
pytest-twisted >= 1.11
|
||||
pytest-xdist
|
||||
|
|
@ -14,6 +13,7 @@ uvloop; platform_system != "Windows"
|
|||
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
brotlipy
|
||||
brotlipy # optional for HTTP compress downloader middleware tests
|
||||
zstandard # optional for HTTP compress downloader middleware tests
|
||||
ipython
|
||||
pywin32; sys_platform == "win32"
|
||||
|
|
|
|||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
|
@ -389,8 +389,9 @@ class GenspiderCommandTest(CommandTest):
|
|||
def test_template(self, tplname='crawl'):
|
||||
args = [f'--template={tplname}'] if tplname else []
|
||||
spname = 'test_spider'
|
||||
spmodule = f"{self.project_name}.spiders.{spname}"
|
||||
p, out, err = self.proc('genspider', spname, 'test.com', *args)
|
||||
self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module", out)
|
||||
self.assertIn(f"Created spider {spname!r} using template {tplname!r} in module:{os.linesep} {spmodule}", out)
|
||||
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
|
||||
modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
|
||||
p, out, err = self.proc('genspider', spname, 'test.com', *args)
|
||||
|
|
@ -496,6 +497,8 @@ class MiscCommandsTest(CommandTest):
|
|||
|
||||
class RunSpiderCommandTest(CommandTest):
|
||||
|
||||
spider_filename = 'myspider.py'
|
||||
|
||||
debug_log_spider = """
|
||||
import scrapy
|
||||
|
||||
|
|
@ -507,11 +510,23 @@ class MySpider(scrapy.Spider):
|
|||
return []
|
||||
"""
|
||||
|
||||
badspider = """
|
||||
import scrapy
|
||||
|
||||
class BadSpider(scrapy.Spider):
|
||||
name = "bad"
|
||||
def start_requests(self):
|
||||
raise Exception("oops!")
|
||||
"""
|
||||
|
||||
@contextmanager
|
||||
def _create_file(self, content, name):
|
||||
def _create_file(self, content, name=None):
|
||||
tmpdir = self.mktemp()
|
||||
os.mkdir(tmpdir)
|
||||
fname = abspath(join(tmpdir, name))
|
||||
if name:
|
||||
fname = abspath(join(tmpdir, name))
|
||||
else:
|
||||
fname = abspath(join(tmpdir, self.spider_filename))
|
||||
with open(fname, 'w') as f:
|
||||
f.write(content)
|
||||
try:
|
||||
|
|
@ -519,12 +534,12 @@ class MySpider(scrapy.Spider):
|
|||
finally:
|
||||
rmtree(tmpdir)
|
||||
|
||||
def runspider(self, code, name='myspider.py', args=()):
|
||||
def runspider(self, code, name=None, args=()):
|
||||
with self._create_file(code, name) as fname:
|
||||
return self.proc('runspider', fname, *args)
|
||||
|
||||
def get_log(self, code, name='myspider.py', args=()):
|
||||
p, stdout, stderr = self.runspider(code, name=name, args=args)
|
||||
def get_log(self, code, name=None, args=()):
|
||||
p, stdout, stderr = self.runspider(code, name, args=args)
|
||||
return stderr
|
||||
|
||||
def test_runspider(self):
|
||||
|
|
@ -556,7 +571,7 @@ class MySpider(scrapy.Spider):
|
|||
# which is intended,
|
||||
# but this should not be because of DNS lookup error
|
||||
# assumption: localhost will resolve in all cases (true?)
|
||||
log = self.get_log("""
|
||||
dnscache_spider = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
|
|
@ -565,23 +580,20 @@ class MySpider(scrapy.Spider):
|
|||
|
||||
def parse(self, response):
|
||||
return {'test': 'value'}
|
||||
""",
|
||||
args=('-s', 'DNSCACHE_ENABLED=False'))
|
||||
print(log)
|
||||
"""
|
||||
log = self.get_log(dnscache_spider, args=('-s', 'DNSCACHE_ENABLED=False'))
|
||||
self.assertNotIn("DNSLookupError", log)
|
||||
self.assertIn("INFO: Spider opened", log)
|
||||
|
||||
def test_runspider_log_short_names(self):
|
||||
log1 = self.get_log(self.debug_log_spider,
|
||||
args=('-s', 'LOG_SHORT_NAMES=1'))
|
||||
print(log1)
|
||||
self.assertIn("[myspider] DEBUG: It Works!", log1)
|
||||
self.assertIn("[scrapy]", log1)
|
||||
self.assertNotIn("[scrapy.core.engine]", log1)
|
||||
|
||||
log2 = self.get_log(self.debug_log_spider,
|
||||
args=('-s', 'LOG_SHORT_NAMES=0'))
|
||||
print(log2)
|
||||
self.assertIn("[myspider] DEBUG: It Works!", log2)
|
||||
self.assertNotIn("[scrapy]", log2)
|
||||
self.assertIn("[scrapy.core.engine]", log2)
|
||||
|
|
@ -599,15 +611,7 @@ class MySpider(scrapy.Spider):
|
|||
self.assertIn('Unable to load', log)
|
||||
|
||||
def test_start_requests_errors(self):
|
||||
log = self.get_log("""
|
||||
import scrapy
|
||||
|
||||
class BadSpider(scrapy.Spider):
|
||||
name = "bad"
|
||||
def start_requests(self):
|
||||
raise Exception("oops!")
|
||||
""", name="badspider.py")
|
||||
print(log)
|
||||
log = self.get_log(self.badspider, name='badspider.py')
|
||||
self.assertIn("start_requests", log)
|
||||
self.assertIn("badspider.py", log)
|
||||
|
||||
|
|
@ -677,9 +681,14 @@ class MySpider(scrapy.Spider):
|
|||
)
|
||||
return []
|
||||
"""
|
||||
with open(os.path.join(self.cwd, "example.json"), "w") as f1:
|
||||
f1.write("not empty")
|
||||
args = ['-O', 'example.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
|
||||
with open(os.path.join(self.cwd, "example.json")) as f2:
|
||||
first_line = f2.readline()
|
||||
self.assertNotEqual(first_line, "not empty")
|
||||
|
||||
def test_output_and_overwrite_output(self):
|
||||
spider_code = """
|
||||
|
|
@ -696,6 +705,54 @@ class MySpider(scrapy.Spider):
|
|||
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
|
||||
|
||||
|
||||
class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
|
||||
|
||||
spider_filename = 'myspider.pyw'
|
||||
|
||||
def setUp(self):
|
||||
super(WindowsRunSpiderCommandTest, self).setUp()
|
||||
|
||||
def test_start_requests_errors(self):
|
||||
log = self.get_log(self.badspider, name='badspider.pyw')
|
||||
self.assertIn("start_requests", log)
|
||||
self.assertIn("badspider.pyw", log)
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_run_good_spider(self):
|
||||
super().test_run_good_spider()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider(self):
|
||||
super().test_runspider()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_dnscache_disabled(self):
|
||||
super().test_runspider_dnscache_disabled()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_log_level(self):
|
||||
super().test_runspider_log_level()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_log_short_names(self):
|
||||
super().test_runspider_log_short_names()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_runspider_no_spider_found(self):
|
||||
super().test_runspider_no_spider_found()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_output(self):
|
||||
super().test_output()
|
||||
|
||||
@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
|
||||
def test_overwrite_output(self):
|
||||
super().test_overwrite_output()
|
||||
|
||||
def test_runspider_unable_to_load(self):
|
||||
raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
|
||||
|
||||
|
||||
class BenchCommandTest(CommandTest):
|
||||
|
||||
def test_run(self):
|
||||
|
|
@ -762,9 +819,14 @@ class MySpider(scrapy.Spider):
|
|||
)
|
||||
return []
|
||||
"""
|
||||
with open(os.path.join(self.cwd, "example.json"), "w") as f1:
|
||||
f1.write("not empty")
|
||||
args = ['-O', 'example.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
|
||||
with open(os.path.join(self.cwd, "example.json")) as f2:
|
||||
first_line = f2.readline()
|
||||
self.assertNotEqual(first_line, "not empty")
|
||||
|
||||
def test_output_and_overwrite_output(self):
|
||||
spider_code = """
|
||||
|
|
|
|||
|
|
@ -22,6 +22,8 @@ from scrapy.extensions.throttle import AutoThrottle
|
|||
from scrapy.extensions import telnet
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
|
||||
class BaseCrawlerTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -280,9 +282,9 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
|
||||
|
||||
class ScriptRunnerMixin:
|
||||
def run_script(self, script_name):
|
||||
def run_script(self, script_name, *script_args):
|
||||
script_path = os.path.join(self.script_dir, script_name)
|
||||
args = (sys.executable, script_path)
|
||||
args = [sys.executable, script_path] + list(script_args)
|
||||
p = subprocess.Popen(args, env=get_testenv(),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
stdout, stderr = p.communicate()
|
||||
|
|
@ -321,11 +323,20 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
"twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.",
|
||||
log)
|
||||
|
||||
def test_ipv6_alternative_name_resolver(self):
|
||||
log = self.run_script('alternative_name_resolver.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
def test_caching_hostname_resolver_ipv6(self):
|
||||
log = self.run_script("caching_hostname_resolver_ipv6.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
|
||||
|
||||
def test_caching_hostname_resolver_finite_execution(self):
|
||||
with MockServer() as mock_server:
|
||||
http_address = mock_server.http_address.replace("0.0.0.0", "127.0.0.1")
|
||||
log = self.run_script("caching_hostname_resolver.py", http_address)
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertNotIn("ERROR: Error downloading", log)
|
||||
self.assertNotIn("TimeoutError", log)
|
||||
self.assertNotIn("twisted.internet.error.DNSLookupError", log)
|
||||
|
||||
def test_reactor_select(self):
|
||||
log = self.run_script("twisted_reactor_select.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
|
|
@ -353,6 +364,25 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
|
||||
@mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
|
||||
def test_custom_loop_asyncio_deferred_signal(self):
|
||||
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
self.assertIn("async pipeline opened!", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_default_loop_asyncio_deferred_signal(self):
|
||||
log = self.run_script("asyncio_deferred_signal.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
self.assertNotIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
self.assertIn("async pipeline opened!", log)
|
||||
|
||||
|
||||
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ class FileTestCase(unittest.TestCase):
|
|||
self.assertEqual(response.url, request.url)
|
||||
self.assertEqual(response.status, 200)
|
||||
self.assertEqual(response.body, b'0123456789')
|
||||
self.assertEqual(response.protocol, None)
|
||||
|
||||
request = Request(path_to_file_uri(self.tmpname + '^'))
|
||||
assert request.url.upper().endswith('%5E')
|
||||
|
|
@ -360,6 +361,13 @@ class Http10TestCase(HttpTestCase):
|
|||
"""HTTP 1.0 test case"""
|
||||
download_handler_cls = HTTP10DownloadHandler
|
||||
|
||||
def test_protocol(self):
|
||||
request = Request(self.getURL("host"), method="GET")
|
||||
d = self.download_request(request, Spider("foo"))
|
||||
d.addCallback(lambda r: r.protocol)
|
||||
d.addCallback(self.assertEqual, "HTTP/1.0")
|
||||
return d
|
||||
|
||||
|
||||
class Https10TestCase(Http10TestCase):
|
||||
scheme = 'https'
|
||||
|
|
@ -489,6 +497,13 @@ class Http11TestCase(HttpTestCase):
|
|||
def test_download_broken_chunked_content_allow_data_loss_via_setting(self):
|
||||
return self.test_download_broken_content_allow_data_loss_via_setting('broken-chunked')
|
||||
|
||||
def test_protocol(self):
|
||||
request = Request(self.getURL("host"), method="GET")
|
||||
d = self.download_request(request, Spider("foo"))
|
||||
d.addCallback(lambda r: r.protocol)
|
||||
d.addCallback(self.assertEqual, "HTTP/1.1")
|
||||
return d
|
||||
|
||||
|
||||
class Https11TestCase(Http11TestCase):
|
||||
scheme = 'https'
|
||||
|
|
@ -868,29 +883,6 @@ class S3TestCase(unittest.TestCase):
|
|||
self.assertEqual(httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
|
||||
|
||||
def test_request_signing5(self):
|
||||
try:
|
||||
import botocore # noqa: F401
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
raise unittest.SkipTest(
|
||||
'botocore does not support overriding date with x-amz-date')
|
||||
# deletes an object from the 'johnsmith' bucket using the
|
||||
# path-style and Date alternative.
|
||||
date = 'Tue, 27 Mar 2007 21:20:27 +0000'
|
||||
req = Request(
|
||||
's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
|
||||
'Date': date,
|
||||
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
|
||||
})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
# botocore does not override Date with x-amz-date
|
||||
self.assertEqual(
|
||||
httpreq.headers['Authorization'],
|
||||
b'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
|
||||
|
||||
def test_request_signing6(self):
|
||||
# uploads an object to a CNAME style virtual hosted bucket with metadata.
|
||||
date = 'Tue, 27 Mar 2007 21:06:08 +0000'
|
||||
|
|
@ -985,6 +977,7 @@ class BaseFTPTestCase(unittest.TestCase):
|
|||
self.assertEqual(r.status, 200)
|
||||
self.assertEqual(r.body, b'I have the power!')
|
||||
self.assertEqual(r.headers, {b'Local Filename': [b''], b'Size': [b'17']})
|
||||
self.assertIsNone(r.protocol)
|
||||
return self._add_test_callbacks(d, _test)
|
||||
|
||||
def test_ftp_download_path_with_spaces(self):
|
||||
|
|
@ -1143,3 +1136,10 @@ class DataURITestCase(unittest.TestCase):
|
|||
|
||||
request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
|
||||
return self.download_request(request, self.spider).addCallback(_test)
|
||||
|
||||
def test_protocol(self):
|
||||
def _test(response):
|
||||
self.assertIsNone(response.protocol)
|
||||
|
||||
request = Request("data:,")
|
||||
return self.download_request(request, self.spider).addCallback(_test)
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import logging
|
|||
from testfixtures import LogCapture
|
||||
from unittest import TestCase
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
|
||||
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
|
||||
from scrapy.exceptions import NotConfigured
|
||||
|
|
@ -243,6 +245,7 @@ class CookiesMiddlewareTest(TestCase):
|
|||
self.assertIn('Cookie', request.headers)
|
||||
self.assertEqual(b'currencyCookie=USD', request.headers['Cookie'])
|
||||
|
||||
@pytest.mark.xfail(reason="Cookie header is not currently being processed")
|
||||
def test_keep_cookie_from_default_request_headers_middleware(self):
|
||||
DEFAULT_REQUEST_HEADERS = dict(Cookie='default=value; asdf=qwerty')
|
||||
mw_default_headers = DefaultHeadersMiddleware(DEFAULT_REQUEST_HEADERS.items())
|
||||
|
|
@ -257,6 +260,7 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert self.mw.process_request(req2, self.spider) is None
|
||||
self.assertCookieValEqual(req2.headers['Cookie'], b'default=value; a=b; asdf=qwerty')
|
||||
|
||||
@pytest.mark.xfail(reason="Cookie header is not currently being processed")
|
||||
def test_keep_cookie_header(self):
|
||||
# keep only cookies from 'Cookie' request header
|
||||
req1 = Request('http://scrapytest.org', headers={'Cookie': 'a=b; c=d'})
|
||||
|
|
@ -291,6 +295,7 @@ class CookiesMiddlewareTest(TestCase):
|
|||
assert self.mw.process_request(req3, self.spider) is None
|
||||
self.assertCookieValEqual(req3.headers['Cookie'], b'a=\xc3\xa1')
|
||||
|
||||
@pytest.mark.xfail(reason="Cookie header is not currently being processed")
|
||||
def test_request_headers_cookie_encoding(self):
|
||||
# 1) UTF8-encoded bytes
|
||||
req1 = Request('http://example.org', headers={'Cookie': 'a=á'.encode('utf8')})
|
||||
|
|
|
|||
|
|
@ -20,6 +20,12 @@ FORMAT = {
|
|||
'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
|
||||
'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
|
||||
'br': ('html-br.bin', 'br'),
|
||||
# $ zstd raw.html --content-size -o html-zstd-static-content-size.bin
|
||||
'zstd-static-content-size': ('html-zstd-static-content-size.bin', 'zstd'),
|
||||
# $ zstd raw.html --no-content-size -o html-zstd-static-no-content-size.bin
|
||||
'zstd-static-no-content-size': ('html-zstd-static-no-content-size.bin', 'zstd'),
|
||||
# $ cat raw.html | zstd -o html-zstd-streaming-no-content-size.bin
|
||||
'zstd-streaming-no-content-size': ('html-zstd-streaming-no-content-size.bin', 'zstd'),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -80,6 +86,27 @@ class HttpCompressionTest(TestCase):
|
|||
assert newresponse.body.startswith(b"<!DOCTYPE")
|
||||
assert 'Content-Encoding' not in newresponse.headers
|
||||
|
||||
def test_process_response_zstd(self):
|
||||
try:
|
||||
import zstandard # noqa: F401
|
||||
except ImportError:
|
||||
raise SkipTest("no zstd support (zstandard)")
|
||||
raw_content = None
|
||||
for check_key in FORMAT:
|
||||
if not check_key.startswith('zstd-'):
|
||||
continue
|
||||
response = self._getresponse(check_key)
|
||||
request = response.request
|
||||
self.assertEqual(response.headers['Content-Encoding'], b'zstd')
|
||||
newresponse = self.mw.process_response(request, response, self.spider)
|
||||
if raw_content is None:
|
||||
raw_content = newresponse.body
|
||||
else:
|
||||
assert raw_content == newresponse.body
|
||||
assert newresponse is not response
|
||||
assert newresponse.body.startswith(b"<!DOCTYPE")
|
||||
assert 'Content-Encoding' not in newresponse.headers
|
||||
|
||||
def test_process_response_rawdeflate(self):
|
||||
response = self._getresponse('rawdeflate')
|
||||
request = response.request
|
||||
|
|
|
|||
|
|
@ -145,3 +145,10 @@ class TestHttpProxyMiddleware(TestCase):
|
|||
req = Request('http://noproxy.com', meta={'proxy': 'http://proxy.com'})
|
||||
assert mw.process_request(req, spider) is None
|
||||
self.assertEqual(req.meta, {'proxy': 'http://proxy.com'})
|
||||
|
||||
def test_no_proxy_invalid_values(self):
|
||||
os.environ['no_proxy'] = '/var/run/docker.sock'
|
||||
mw = HttpProxyMiddleware()
|
||||
# '/var/run/docker.sock' may be used by the user for
|
||||
# no_proxy value but is not parseable and should be skipped
|
||||
assert 'no' not in mw.proxies
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
def test_df_from_crawler_scheduler(self):
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertTrue(scheduler.df.debug)
|
||||
|
|
@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
def test_df_from_settings_scheduler(self):
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertTrue(scheduler.df.debug)
|
||||
self.assertEqual(scheduler.df.method, 'from_settings')
|
||||
|
||||
def test_df_direct_scheduler(self):
|
||||
settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
|
||||
settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertEqual(scheduler.df.method, 'n/a')
|
||||
|
|
@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_log(self):
|
||||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': False,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
|
@ -191,7 +191,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_log_debug(self):
|
||||
with LogCapture() as log:
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
|
|
|||
|
|
@ -8,12 +8,13 @@ import tempfile
|
|||
import warnings
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from contextlib import ExitStack
|
||||
from io import BytesIO
|
||||
from logging import getLogger
|
||||
from pathlib import Path
|
||||
from string import ascii_letters, digits
|
||||
from unittest import mock
|
||||
from urllib.parse import urljoin, urlparse, quote
|
||||
from urllib.parse import urljoin, quote
|
||||
from urllib.request import pathname2url
|
||||
|
||||
import lxml.etree
|
||||
|
|
@ -41,14 +42,27 @@ from scrapy.extensions.feedexport import (
|
|||
from scrapy.settings import Settings
|
||||
from scrapy.utils.python import to_unicode
|
||||
from scrapy.utils.test import (
|
||||
assert_aws_environ,
|
||||
get_s3_content_and_delete,
|
||||
get_crawler,
|
||||
mock_google_cloud_storage,
|
||||
skip_if_no_boto,
|
||||
)
|
||||
|
||||
from tests.mockserver import MockFTPServer, MockServer
|
||||
from tests.spiders import ItemSpider
|
||||
|
||||
|
||||
def path_to_url(path):
|
||||
return urljoin('file:', pathname2url(str(path)))
|
||||
|
||||
|
||||
def printf_escape(string):
|
||||
return string.replace('%', '%%')
|
||||
|
||||
|
||||
def build_url(path):
|
||||
if path[0] != '/':
|
||||
path = '/' + path
|
||||
return urljoin('file:', path)
|
||||
|
||||
|
||||
class FileFeedStorageTest(unittest.TestCase):
|
||||
|
|
@ -254,21 +268,42 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
|
||||
@defer.inlineCallbacks
|
||||
def test_store(self):
|
||||
assert_aws_environ()
|
||||
uri = os.environ.get('S3_TEST_FILE_URI')
|
||||
if not uri:
|
||||
raise unittest.SkipTest("No S3 URI available for testing")
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
storage = S3FeedStorage(uri, access_key, secret_key)
|
||||
skip_if_no_boto()
|
||||
|
||||
settings = {
|
||||
'AWS_ACCESS_KEY_ID': 'access_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'secret_key',
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
bucket = 'mybucket'
|
||||
key = 'export.csv'
|
||||
storage = S3FeedStorage.from_crawler(crawler, f's3://{bucket}/{key}')
|
||||
verifyObject(IFeedStorage, storage)
|
||||
file = storage.open(scrapy.Spider("default"))
|
||||
expected_content = b"content: \xe2\x98\x83"
|
||||
file.write(expected_content)
|
||||
yield storage.store(file)
|
||||
u = urlparse(uri)
|
||||
content = get_s3_content_and_delete(u.hostname, u.path[1:])
|
||||
self.assertEqual(content, expected_content)
|
||||
|
||||
file = mock.MagicMock()
|
||||
from botocore.stub import Stubber
|
||||
with Stubber(storage.s3_client) as stub:
|
||||
stub.add_response(
|
||||
'put_object',
|
||||
expected_params={
|
||||
'Body': file,
|
||||
'Bucket': bucket,
|
||||
'Key': key,
|
||||
},
|
||||
service_response={},
|
||||
)
|
||||
|
||||
yield storage.store(file)
|
||||
|
||||
stub.assert_no_pending_responses()
|
||||
self.assertEqual(
|
||||
file.method_calls,
|
||||
[
|
||||
mock.call.seek(0),
|
||||
# The call to read does not happen with Stubber
|
||||
mock.call.close(),
|
||||
]
|
||||
)
|
||||
|
||||
def test_init_without_acl(self):
|
||||
storage = S3FeedStorage(
|
||||
|
|
@ -601,12 +636,6 @@ class FeedExportTest(FeedExportTestBase):
|
|||
def run_and_export(self, spider_cls, settings):
|
||||
""" Run spider with specified settings; return exported data. """
|
||||
|
||||
def path_to_url(path):
|
||||
return urljoin('file:', pathname2url(str(path)))
|
||||
|
||||
def printf_escape(string):
|
||||
return string.replace('%', '%%')
|
||||
|
||||
FEEDS = settings.get('FEEDS') or {}
|
||||
settings['FEEDS'] = {
|
||||
printf_escape(path_to_url(file_path)): feed_options
|
||||
|
|
@ -729,6 +758,69 @@ class FeedExportTest(FeedExportTestBase):
|
|||
result = self._load_until_eof(data['marshal'], load_func=marshal.load)
|
||||
self.assertEqual(expected, result)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_stats_file_success(self):
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
printf_escape(path_to_url(self._random_temp_filename())): {
|
||||
"format": "json",
|
||||
}
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
with MockServer() as mockserver:
|
||||
yield crawler.crawl(mockserver=mockserver)
|
||||
self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
|
||||
self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_stats_file_failed(self):
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
printf_escape(path_to_url(self._random_temp_filename())): {
|
||||
"format": "json",
|
||||
}
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
with ExitStack() as stack:
|
||||
mockserver = stack.enter_context(MockServer())
|
||||
stack.enter_context(
|
||||
mock.patch(
|
||||
"scrapy.extensions.feedexport.FileFeedStorage.store",
|
||||
side_effect=KeyError("foo"))
|
||||
)
|
||||
yield crawler.crawl(mockserver=mockserver)
|
||||
self.assertIn("feedexport/failed_count/FileFeedStorage", crawler.stats.get_stats())
|
||||
self.assertEqual(crawler.stats.get_value("feedexport/failed_count/FileFeedStorage"), 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_stats_multiple_file(self):
|
||||
settings = {
|
||||
'AWS_ACCESS_KEY_ID': 'access_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'secret_key',
|
||||
"FEEDS": {
|
||||
printf_escape(path_to_url(self._random_temp_filename())): {
|
||||
"format": "json",
|
||||
},
|
||||
"s3://bucket/key/foo.csv": {
|
||||
"format": "csv",
|
||||
},
|
||||
"stdout:": {
|
||||
"format": "xml",
|
||||
}
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
with MockServer() as mockserver, mock.patch.object(S3FeedStorage, "store"):
|
||||
yield crawler.crawl(mockserver=mockserver)
|
||||
self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
|
||||
self.assertIn("feedexport/success_count/S3FeedStorage", crawler.stats.get_stats())
|
||||
self.assertIn("feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats())
|
||||
self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1)
|
||||
self.assertEqual(crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1)
|
||||
self.assertEqual(crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_export_items(self):
|
||||
# feed exporters use field names from Item
|
||||
|
|
@ -1191,6 +1283,43 @@ class FeedExportTest(FeedExportTestBase):
|
|||
for fmt in ['json', 'xml', 'csv']:
|
||||
self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_extend_kwargs(self):
|
||||
items = [{'foo': 'FOO', 'bar': 'BAR'}]
|
||||
|
||||
expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
|
||||
expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
|
||||
test_cases = [
|
||||
# with title
|
||||
{
|
||||
'options': {
|
||||
'format': 'csv',
|
||||
'item_export_kwargs': {'include_headers_line': True},
|
||||
},
|
||||
'expected': expected_with_title_csv,
|
||||
},
|
||||
# without title
|
||||
{
|
||||
'options': {
|
||||
'format': 'csv',
|
||||
'item_export_kwargs': {'include_headers_line': False},
|
||||
},
|
||||
'expected': expected_without_title_csv,
|
||||
},
|
||||
]
|
||||
|
||||
for row in test_cases:
|
||||
feed_options = row['options']
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
self._random_temp_filename(): feed_options,
|
||||
},
|
||||
'FEED_EXPORT_INDENT': None,
|
||||
}
|
||||
|
||||
data = yield self.exported_data(items, settings)
|
||||
self.assertEqual(row['expected'], data[feed_options['format']])
|
||||
|
||||
|
||||
class BatchDeliveriesTest(FeedExportTestBase):
|
||||
__test__ = True
|
||||
|
|
@ -1200,11 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
def run_and_export(self, spider_cls, settings):
|
||||
""" Run spider with specified settings; return exported data. """
|
||||
|
||||
def build_url(path):
|
||||
if path[0] != '/':
|
||||
path = '/' + path
|
||||
return urljoin('file:', path)
|
||||
|
||||
FEEDS = settings.get('FEEDS') or {}
|
||||
settings['FEEDS'] = {
|
||||
build_url(file_path): feed
|
||||
|
|
@ -1495,47 +1619,70 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
self.assertEqual(len(items) + 1, len(data['json']))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_s3_export(self):
|
||||
"""
|
||||
Test export of items into s3 bucket.
|
||||
S3_TEST_BUCKET_NAME, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY must be specified in tox.ini
|
||||
to perform this test:
|
||||
[testenv]
|
||||
setenv =
|
||||
AWS_SECRET_ACCESS_KEY = ABCD
|
||||
AWS_ACCESS_KEY_ID = EFGH
|
||||
S3_TEST_BUCKET_NAME = IJKL
|
||||
"""
|
||||
try:
|
||||
import boto3
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("S3FeedStorage requires boto3")
|
||||
|
||||
assert_aws_environ()
|
||||
s3_test_bucket_name = os.environ.get('S3_TEST_BUCKET_NAME')
|
||||
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
||||
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
||||
if not s3_test_bucket_name:
|
||||
raise unittest.SkipTest("No S3 BUCKET available for testing")
|
||||
|
||||
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
|
||||
filename = ''.join(chars)
|
||||
prefix = f'tmp/{filename}'
|
||||
s3_test_file_uri = f's3://{s3_test_bucket_name}/{prefix}/%(batch_time)s.json'
|
||||
storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)
|
||||
settings = Settings({
|
||||
'FEEDS': {
|
||||
s3_test_file_uri: {
|
||||
'format': 'json',
|
||||
},
|
||||
def test_stats_batch_file_success(self):
|
||||
settings = {
|
||||
"FEEDS": {
|
||||
build_url(os.path.join(self._random_temp_filename(), "json", self._file_mark)): {
|
||||
"format": "json",
|
||||
}
|
||||
},
|
||||
'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
|
||||
})
|
||||
"FEED_EXPORT_BATCH_ITEM_COUNT": 1,
|
||||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
with MockServer() as mockserver:
|
||||
yield crawler.crawl(total=2, mockserver=mockserver)
|
||||
self.assertIn("feedexport/success_count/FileFeedStorage", crawler.stats.get_stats())
|
||||
self.assertEqual(crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 12)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_s3_export(self):
|
||||
skip_if_no_boto()
|
||||
|
||||
bucket = 'mybucket'
|
||||
items = [
|
||||
self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
|
||||
self.MyItem({'foo': 'bar2', 'egg': 'spam2', 'baz': 'quux2'}),
|
||||
self.MyItem({'foo': 'bar3', 'baz': 'quux3'}),
|
||||
]
|
||||
|
||||
class CustomS3FeedStorage(S3FeedStorage):
|
||||
|
||||
stubs = []
|
||||
|
||||
def open(self, *args, **kwargs):
|
||||
from botocore.stub import ANY, Stubber
|
||||
stub = Stubber(self.s3_client)
|
||||
stub.activate()
|
||||
CustomS3FeedStorage.stubs.append(stub)
|
||||
stub.add_response(
|
||||
'put_object',
|
||||
expected_params={
|
||||
'Body': ANY,
|
||||
'Bucket': bucket,
|
||||
'Key': ANY,
|
||||
},
|
||||
service_response={},
|
||||
)
|
||||
return super().open(*args, **kwargs)
|
||||
|
||||
key = 'export.csv'
|
||||
uri = f's3://{bucket}/{key}/%(batch_time)s.json'
|
||||
batch_item_count = 1
|
||||
settings = {
|
||||
'AWS_ACCESS_KEY_ID': 'access_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'secret_key',
|
||||
'FEED_EXPORT_BATCH_ITEM_COUNT': batch_item_count,
|
||||
'FEED_STORAGES': {
|
||||
's3': CustomS3FeedStorage,
|
||||
},
|
||||
'FEEDS': {
|
||||
uri: {
|
||||
'format': 'json',
|
||||
},
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
storage = S3FeedStorage.from_crawler(crawler, uri)
|
||||
verifyObject(IFeedStorage, storage)
|
||||
|
||||
class TestSpider(scrapy.Spider):
|
||||
|
|
@ -1545,22 +1692,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
for item in items:
|
||||
yield item
|
||||
|
||||
s3 = boto3.resource('s3')
|
||||
my_bucket = s3.Bucket(s3_test_bucket_name)
|
||||
batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
|
||||
|
||||
with MockServer() as s:
|
||||
with MockServer() as server:
|
||||
runner = CrawlerRunner(Settings(settings))
|
||||
TestSpider.start_urls = [s.url('/')]
|
||||
TestSpider.start_urls = [server.url('/')]
|
||||
yield runner.crawl(TestSpider)
|
||||
|
||||
for file_uri in my_bucket.objects.filter(Prefix=prefix):
|
||||
content = get_s3_content_and_delete(s3_test_bucket_name, file_uri.key)
|
||||
if not content and not items:
|
||||
break
|
||||
content = json.loads(content.decode('utf-8'))
|
||||
expected_batch, items = items[:batch_size], items[batch_size:]
|
||||
self.assertEqual(expected_batch, content)
|
||||
self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
|
||||
for stub in CustomS3FeedStorage.stubs[:-1]:
|
||||
stub.assert_no_pending_responses()
|
||||
|
||||
|
||||
class FeedExportInitTest(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -43,6 +43,15 @@ class RequestTest(unittest.TestCase):
|
|||
assert r.headers is not headers
|
||||
self.assertEqual(r.headers[b"caca"], b"coco")
|
||||
|
||||
def test_url_scheme(self):
|
||||
# This test passes by not raising any (ValueError) exception
|
||||
self.request_class('http://example.org')
|
||||
self.request_class('https://example.org')
|
||||
self.request_class('s3://example.org')
|
||||
self.request_class('ftp://example.org')
|
||||
self.request_class('about:config')
|
||||
self.request_class('data:,Hello%2C%20World!')
|
||||
|
||||
def test_url_no_scheme(self):
|
||||
self.assertRaises(ValueError, self.request_class, 'foo')
|
||||
self.assertRaises(ValueError, self.request_class, '/foo/')
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
|
|||
self.base_settings = {
|
||||
'LOG_LEVEL': 'DEBUG',
|
||||
'ITEM_PIPELINES': {
|
||||
__name__ + '.DropSomeItemsPipeline': 300,
|
||||
DropSomeItemsPipeline: 300,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -212,7 +212,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
|
|||
@defer.inlineCallbacks
|
||||
def test_skip_messages(self):
|
||||
settings = self.base_settings.copy()
|
||||
settings['LOG_FORMATTER'] = __name__ + '.SkipMessagesLogFormatter'
|
||||
settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
|
||||
crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
|
||||
with LogCapture() as lc:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime
|
||||
from io import BytesIO
|
||||
from shutil import rmtree
|
||||
from tempfile import mkdtemp
|
||||
|
|
@ -23,11 +24,10 @@ from scrapy.pipelines.files import (
|
|||
)
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.test import (
|
||||
assert_aws_environ,
|
||||
assert_gcs_environ,
|
||||
get_ftp_content_and_delete,
|
||||
get_gcs_content_and_delete,
|
||||
get_s3_content_and_delete,
|
||||
skip_if_no_boto,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -414,32 +414,88 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
|
||||
|
||||
class TestS3FilesStore(unittest.TestCase):
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_persist(self):
|
||||
assert_aws_environ()
|
||||
uri = os.environ.get('S3_TEST_FILE_URI')
|
||||
if not uri:
|
||||
raise unittest.SkipTest("No S3 URI available for testing")
|
||||
data = b"TestS3FilesStore: \xe2\x98\x83"
|
||||
buf = BytesIO(data)
|
||||
skip_if_no_boto()
|
||||
|
||||
bucket = 'mybucket'
|
||||
key = 'export.csv'
|
||||
uri = f's3://{bucket}/{key}'
|
||||
buffer = mock.MagicMock()
|
||||
meta = {'foo': 'bar'}
|
||||
path = ''
|
||||
content_type = 'image/png'
|
||||
|
||||
store = S3FilesStore(uri)
|
||||
yield store.persist_file(
|
||||
path, buf, info=None, meta=meta,
|
||||
headers={'Content-Type': 'image/png'})
|
||||
s = yield store.stat_file(path, info=None)
|
||||
self.assertIn('last_modified', s)
|
||||
self.assertIn('checksum', s)
|
||||
self.assertEqual(s['checksum'], '3187896a9657a28163abb31667df64c8')
|
||||
u = urlparse(uri)
|
||||
content, key = get_s3_content_and_delete(
|
||||
u.hostname, u.path[1:], with_key=True)
|
||||
self.assertEqual(content, data)
|
||||
self.assertEqual(key['Metadata'], {'foo': 'bar'})
|
||||
self.assertEqual(
|
||||
key['CacheControl'], S3FilesStore.HEADERS['Cache-Control'])
|
||||
self.assertEqual(key['ContentType'], 'image/png')
|
||||
from botocore.stub import Stubber
|
||||
with Stubber(store.s3_client) as stub:
|
||||
stub.add_response(
|
||||
'put_object',
|
||||
expected_params={
|
||||
'ACL': S3FilesStore.POLICY,
|
||||
'Body': buffer,
|
||||
'Bucket': bucket,
|
||||
'CacheControl': S3FilesStore.HEADERS['Cache-Control'],
|
||||
'ContentType': content_type,
|
||||
'Key': key,
|
||||
'Metadata': meta,
|
||||
},
|
||||
service_response={},
|
||||
)
|
||||
|
||||
yield store.persist_file(
|
||||
path,
|
||||
buffer,
|
||||
info=None,
|
||||
meta=meta,
|
||||
headers={'Content-Type': content_type},
|
||||
)
|
||||
|
||||
stub.assert_no_pending_responses()
|
||||
self.assertEqual(
|
||||
buffer.method_calls,
|
||||
[
|
||||
mock.call.seek(0),
|
||||
# The call to read does not happen with Stubber
|
||||
]
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_stat(self):
|
||||
skip_if_no_boto()
|
||||
|
||||
bucket = 'mybucket'
|
||||
key = 'export.csv'
|
||||
uri = f's3://{bucket}/{key}'
|
||||
checksum = '3187896a9657a28163abb31667df64c8'
|
||||
last_modified = datetime(2019, 12, 1)
|
||||
|
||||
store = S3FilesStore(uri)
|
||||
from botocore.stub import Stubber
|
||||
with Stubber(store.s3_client) as stub:
|
||||
stub.add_response(
|
||||
'head_object',
|
||||
expected_params={
|
||||
'Bucket': bucket,
|
||||
'Key': key,
|
||||
},
|
||||
service_response={
|
||||
'ETag': f'"{checksum}"',
|
||||
'LastModified': last_modified,
|
||||
},
|
||||
)
|
||||
|
||||
file_stats = yield store.stat_file('', info=None)
|
||||
self.assertEqual(
|
||||
file_stats,
|
||||
{
|
||||
'checksum': checksum,
|
||||
'last_modified': last_modified.timestamp(),
|
||||
},
|
||||
)
|
||||
|
||||
stub.assert_no_pending_responses()
|
||||
|
||||
|
||||
class TestGCSFilesStore(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -68,7 +68,7 @@ class PipelineTestCase(unittest.TestCase):
|
|||
|
||||
def _create_crawler(self, pipeline_class):
|
||||
settings = {
|
||||
'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
|
||||
'ITEM_PIPELINES': {pipeline_class: 1},
|
||||
}
|
||||
crawler = get_crawler(ItemSpider, settings)
|
||||
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ class CrawlTestCase(TestCase):
|
|||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".RaiseExceptionRequestMiddleware": 590,
|
||||
RaiseExceptionRequestMiddleware: 590,
|
||||
},
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
|
|
@ -119,7 +119,7 @@ class CrawlTestCase(TestCase):
|
|||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".ProcessResponseMiddleware": 595,
|
||||
ProcessResponseMiddleware: 595,
|
||||
}
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
|
|
@ -149,8 +149,8 @@ class CrawlTestCase(TestCase):
|
|||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".RaiseExceptionRequestMiddleware": 590,
|
||||
__name__ + ".CatchExceptionOverrideRequestMiddleware": 595,
|
||||
RaiseExceptionRequestMiddleware: 590,
|
||||
CatchExceptionOverrideRequestMiddleware: 595,
|
||||
},
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
|
|
@ -170,8 +170,8 @@ class CrawlTestCase(TestCase):
|
|||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".RaiseExceptionRequestMiddleware": 590,
|
||||
__name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,
|
||||
RaiseExceptionRequestMiddleware: 590,
|
||||
CatchExceptionDoNotOverrideRequestMiddleware: 595,
|
||||
},
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
|
|
@ -188,7 +188,7 @@ class CrawlTestCase(TestCase):
|
|||
"""
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".AlternativeCallbacksMiddleware": 595,
|
||||
AlternativeCallbacksMiddleware: 595,
|
||||
}
|
||||
})
|
||||
crawler = runner.create_crawler(AlternativeCallbacksSpider)
|
||||
|
|
|
|||
|
|
@ -50,10 +50,10 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
name = 'kwargs'
|
||||
custom_settings = {
|
||||
'DOWNLOADER_MIDDLEWARES': {
|
||||
__name__ + '.InjectArgumentsDownloaderMiddleware': 750,
|
||||
InjectArgumentsDownloaderMiddleware: 750,
|
||||
},
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.InjectArgumentsSpiderMiddleware': 750,
|
||||
InjectArgumentsSpiderMiddleware: 750,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
import gzip
|
||||
import inspect
|
||||
from unittest import mock
|
||||
import warnings
|
||||
from io import BytesIO
|
||||
from unittest import mock
|
||||
|
||||
from testfixtures import LogCapture
|
||||
from twisted.trial import unittest
|
||||
|
|
@ -20,7 +20,6 @@ from scrapy.spiders import (
|
|||
XMLFeedSpider,
|
||||
)
|
||||
from scrapy.linkextractors import LinkExtractor
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
@ -280,7 +279,7 @@ class CrawlSpiderTest(SpiderTest):
|
|||
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body)
|
||||
|
||||
def process_request_change_domain(request):
|
||||
def process_request_change_domain(request, response):
|
||||
return request.replace(url=request.url.replace('.org', '.com'))
|
||||
|
||||
class _CrawlSpider(self.spider_class):
|
||||
|
|
@ -290,17 +289,14 @@ class CrawlSpiderTest(SpiderTest):
|
|||
Rule(LinkExtractor(), process_request=process_request_change_domain),
|
||||
)
|
||||
|
||||
with warnings.catch_warnings(record=True) as cw:
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
|
||||
self.assertEqual([r.url for r in output],
|
||||
['http://example.com/somepage/item/12.html',
|
||||
'http://example.com/about.html',
|
||||
'http://example.com/nofollow.html'])
|
||||
self.assertEqual(len(cw), 1)
|
||||
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
|
||||
self.assertEqual([r.url for r in output],
|
||||
['http://example.com/somepage/item/12.html',
|
||||
'http://example.com/about.html',
|
||||
'http://example.com/nofollow.html'])
|
||||
|
||||
def test_process_request_with_response(self):
|
||||
|
||||
|
|
@ -339,20 +335,17 @@ class CrawlSpiderTest(SpiderTest):
|
|||
Rule(LinkExtractor(), process_request='process_request_upper'),
|
||||
)
|
||||
|
||||
def process_request_upper(self, request):
|
||||
def process_request_upper(self, request, response):
|
||||
return request.replace(url=request.url.upper())
|
||||
|
||||
with warnings.catch_warnings(record=True) as cw:
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
|
||||
self.assertEqual([r.url for r in output],
|
||||
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
|
||||
'http://EXAMPLE.ORG/ABOUT.HTML',
|
||||
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
|
||||
self.assertEqual(len(cw), 1)
|
||||
self.assertEqual(cw[0].category, ScrapyDeprecationWarning)
|
||||
spider = _CrawlSpider()
|
||||
output = list(spider._requests_to_follow(response))
|
||||
self.assertEqual(len(output), 3)
|
||||
self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
|
||||
self.assertEqual([r.url for r in output],
|
||||
['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
|
||||
'http://EXAMPLE.ORG/ABOUT.HTML',
|
||||
'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
|
||||
|
||||
def test_process_request_instance_method_with_response(self):
|
||||
|
||||
|
|
|
|||
|
|
@ -16,11 +16,20 @@ class LogExceptionMiddleware:
|
|||
|
||||
# ================================================================================
|
||||
# (0) recover from an exception on a spider callback
|
||||
class RecoveryMiddleware:
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
|
||||
return [
|
||||
{'from': 'process_spider_exception'},
|
||||
Request(response.url, meta={'dont_fail': True}, dont_filter=True),
|
||||
]
|
||||
|
||||
|
||||
class RecoverySpider(Spider):
|
||||
name = 'RecoverySpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.RecoveryMiddleware': 10,
|
||||
RecoveryMiddleware: 10,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -34,15 +43,6 @@ class RecoverySpider(Spider):
|
|||
raise TabError()
|
||||
|
||||
|
||||
class RecoveryMiddleware:
|
||||
def process_spider_exception(self, response, exception, spider):
|
||||
spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
|
||||
return [
|
||||
{'from': 'process_spider_exception'},
|
||||
Request(response.url, meta={'dont_fail': True}, dont_filter=True),
|
||||
]
|
||||
|
||||
|
||||
# ================================================================================
|
||||
# (1) exceptions from a spider middleware's process_spider_input method
|
||||
class FailProcessSpiderInputMiddleware:
|
||||
|
|
@ -56,9 +56,8 @@ class ProcessSpiderInputSpiderWithoutErrback(Spider):
|
|||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
# spider
|
||||
__name__ + '.LogExceptionMiddleware': 10,
|
||||
__name__ + '.FailProcessSpiderInputMiddleware': 8,
|
||||
__name__ + '.LogExceptionMiddleware': 6,
|
||||
FailProcessSpiderInputMiddleware: 8,
|
||||
LogExceptionMiddleware: 6,
|
||||
# engine
|
||||
}
|
||||
}
|
||||
|
|
@ -87,7 +86,7 @@ class GeneratorCallbackSpider(Spider):
|
|||
name = 'GeneratorCallbackSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.LogExceptionMiddleware': 10,
|
||||
LogExceptionMiddleware: 10,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -106,7 +105,7 @@ class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider)
|
|||
name = 'GeneratorCallbackSpiderMiddlewareRightAfterSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.LogExceptionMiddleware': 100000,
|
||||
LogExceptionMiddleware: 100000,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +116,7 @@ class NotGeneratorCallbackSpider(Spider):
|
|||
name = 'NotGeneratorCallbackSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.LogExceptionMiddleware': 10,
|
||||
LogExceptionMiddleware: 10,
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -134,32 +133,13 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
|
|||
name = 'NotGeneratorCallbackSpiderMiddlewareRightAfterSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.LogExceptionMiddleware': 100000,
|
||||
LogExceptionMiddleware: 100000,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ================================================================================
|
||||
# (4) exceptions from a middleware process_spider_output method (generator)
|
||||
class GeneratorOutputChainSpider(Spider):
|
||||
name = 'GeneratorOutputChainSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.GeneratorFailMiddleware': 10,
|
||||
__name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8,
|
||||
__name__ + '.GeneratorRecoverMiddleware': 5,
|
||||
__name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3,
|
||||
},
|
||||
}
|
||||
|
||||
def start_requests(self):
|
||||
yield Request(self.mockserver.url('/status?n=200'))
|
||||
|
||||
def parse(self, response):
|
||||
yield {'processed': ['parse-first-item']}
|
||||
yield {'processed': ['parse-second-item']}
|
||||
|
||||
|
||||
class _GeneratorDoNothingMiddleware:
|
||||
def process_spider_output(self, response, result, spider):
|
||||
for r in result:
|
||||
|
|
@ -205,26 +185,28 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
|
|||
pass
|
||||
|
||||
|
||||
# ================================================================================
|
||||
# (5) exceptions from a middleware process_spider_output method (not generator)
|
||||
class NotGeneratorOutputChainSpider(Spider):
|
||||
name = 'NotGeneratorOutputChainSpider'
|
||||
class GeneratorOutputChainSpider(Spider):
|
||||
name = 'GeneratorOutputChainSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
__name__ + '.NotGeneratorFailMiddleware': 10,
|
||||
__name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8,
|
||||
__name__ + '.NotGeneratorRecoverMiddleware': 5,
|
||||
__name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3,
|
||||
GeneratorFailMiddleware: 10,
|
||||
GeneratorDoNothingAfterFailureMiddleware: 8,
|
||||
GeneratorRecoverMiddleware: 5,
|
||||
GeneratorDoNothingAfterRecoveryMiddleware: 3,
|
||||
},
|
||||
}
|
||||
|
||||
def start_requests(self):
|
||||
return [Request(self.mockserver.url('/status?n=200'))]
|
||||
yield Request(self.mockserver.url('/status?n=200'))
|
||||
|
||||
def parse(self, response):
|
||||
return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
|
||||
yield {'processed': ['parse-first-item']}
|
||||
yield {'processed': ['parse-second-item']}
|
||||
|
||||
|
||||
# ================================================================================
|
||||
# (5) exceptions from a middleware process_spider_output method (not generator)
|
||||
|
||||
class _NotGeneratorDoNothingMiddleware:
|
||||
def process_spider_output(self, response, result, spider):
|
||||
out = []
|
||||
|
|
@ -276,6 +258,24 @@ class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddlew
|
|||
pass
|
||||
|
||||
|
||||
class NotGeneratorOutputChainSpider(Spider):
|
||||
name = 'NotGeneratorOutputChainSpider'
|
||||
custom_settings = {
|
||||
'SPIDER_MIDDLEWARES': {
|
||||
NotGeneratorFailMiddleware: 10,
|
||||
NotGeneratorDoNothingAfterFailureMiddleware: 8,
|
||||
NotGeneratorRecoverMiddleware: 5,
|
||||
NotGeneratorDoNothingAfterRecoveryMiddleware: 3,
|
||||
},
|
||||
}
|
||||
|
||||
def start_requests(self):
|
||||
return [Request(self.mockserver.url('/status?n=200'))]
|
||||
|
||||
def parse(self, response):
|
||||
return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}]
|
||||
|
||||
|
||||
# ================================================================================
|
||||
class TestSpiderMiddleware(TestCase):
|
||||
@classmethod
|
||||
|
|
|
|||
|
|
@ -176,6 +176,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
|
|||
"store_empty": True,
|
||||
"uri_params": (1, 2, 3, 4),
|
||||
"batch_item_count": 2,
|
||||
"item_export_kwargs": dict(),
|
||||
})
|
||||
|
||||
def test_feed_complete_default_values_from_settings_non_empty(self):
|
||||
|
|
@ -198,6 +199,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
|
|||
"store_empty": True,
|
||||
"uri_params": None,
|
||||
"batch_item_count": 2,
|
||||
"item_export_kwargs": dict(),
|
||||
})
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,10 +3,11 @@ from os.path import join
|
|||
|
||||
from w3lib.encoding import html_to_unicode
|
||||
|
||||
from scrapy.utils.gz import gunzip, is_gzipped
|
||||
from scrapy.http import Response, Headers
|
||||
from scrapy.utils.gz import gunzip, gzip_magic_number
|
||||
from scrapy.http import Response
|
||||
from tests import tests_datadir
|
||||
|
||||
|
||||
SAMPLEDIR = join(tests_datadir, 'compressed')
|
||||
|
||||
|
||||
|
|
@ -14,8 +15,12 @@ class GunzipTest(unittest.TestCase):
|
|||
|
||||
def test_gunzip_basic(self):
|
||||
with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
|
||||
text = gunzip(f.read())
|
||||
self.assertEqual(len(text), 9950)
|
||||
r1 = Response("http://www.example.com", body=f.read())
|
||||
self.assertTrue(gzip_magic_number(r1))
|
||||
|
||||
r2 = Response("http://www.example.com", body=gunzip(r1.body))
|
||||
self.assertFalse(gzip_magic_number(r2))
|
||||
self.assertEqual(len(r2.body), 9950)
|
||||
|
||||
def test_gunzip_truncated(self):
|
||||
with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
|
||||
|
|
@ -28,46 +33,16 @@ class GunzipTest(unittest.TestCase):
|
|||
|
||||
def test_gunzip_truncated_short(self):
|
||||
with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
|
||||
text = gunzip(f.read())
|
||||
assert text.endswith(b'</html>')
|
||||
r1 = Response("http://www.example.com", body=f.read())
|
||||
self.assertTrue(gzip_magic_number(r1))
|
||||
|
||||
def test_is_x_gzipped_right(self):
|
||||
hdrs = Headers({"Content-Type": "application/x-gzip"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertTrue(is_gzipped(r1))
|
||||
|
||||
def test_is_gzipped_right(self):
|
||||
hdrs = Headers({"Content-Type": "application/gzip"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertTrue(is_gzipped(r1))
|
||||
|
||||
def test_is_gzipped_not_quite(self):
|
||||
hdrs = Headers({"Content-Type": "application/gzippppp"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertFalse(is_gzipped(r1))
|
||||
|
||||
def test_is_gzipped_case_insensitive(self):
|
||||
hdrs = Headers({"Content-Type": "Application/X-Gzip"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertTrue(is_gzipped(r1))
|
||||
|
||||
hdrs = Headers({"Content-Type": "application/X-GZIP ; charset=utf-8"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertTrue(is_gzipped(r1))
|
||||
r2 = Response("http://www.example.com", body=gunzip(r1.body))
|
||||
assert r2.body.endswith(b'</html>')
|
||||
self.assertFalse(gzip_magic_number(r2))
|
||||
|
||||
def test_is_gzipped_empty(self):
|
||||
r1 = Response("http://www.example.com")
|
||||
self.assertFalse(is_gzipped(r1))
|
||||
|
||||
def test_is_gzipped_wrong(self):
|
||||
hdrs = Headers({"Content-Type": "application/javascript"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertFalse(is_gzipped(r1))
|
||||
|
||||
def test_is_gzipped_with_charset(self):
|
||||
hdrs = Headers({"Content-Type": "application/x-gzip;charset=utf-8"})
|
||||
r1 = Response("http://www.example.com", headers=hdrs)
|
||||
self.assertTrue(is_gzipped(r1))
|
||||
self.assertFalse(gzip_magic_number(r1))
|
||||
|
||||
def test_gunzip_illegal_eof(self):
|
||||
with open(join(SAMPLEDIR, 'unexpected-eof.gz'), 'rb') as f:
|
||||
|
|
|
|||
|
|
@ -1,19 +0,0 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.utils.http import decode_chunked_transfer
|
||||
|
||||
|
||||
class ChunkedTest(unittest.TestCase):
|
||||
|
||||
def test_decode_chunked_transfer(self):
|
||||
"""Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding"""
|
||||
chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n"
|
||||
chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n"
|
||||
chunked_body += "3\r\n" + "con\r\n"
|
||||
chunked_body += "8\r\n" + "sequence\r\n"
|
||||
chunked_body += "0\r\n\r\n"
|
||||
body = decode_chunked_transfer(chunked_body)
|
||||
self.assertEqual(
|
||||
body,
|
||||
"This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
|
||||
)
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import os
|
||||
|
||||
from pytest import mark
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
|
||||
|
|
@ -134,7 +135,6 @@ class XmliterTestCase(unittest.TestCase):
|
|||
"""
|
||||
response = XmlResponse(url='http://mydummycompany.com', body=body)
|
||||
my_iter = self.xmliter(response, 'item')
|
||||
|
||||
node = next(my_iter)
|
||||
node.register_namespace('g', 'http://base.google.com/ns/1.0')
|
||||
self.assertEqual(node.xpath('title/text()').getall(), ['Item 1'])
|
||||
|
|
@ -150,6 +150,55 @@ class XmliterTestCase(unittest.TestCase):
|
|||
self.assertEqual(node.xpath('id/text()').getall(), [])
|
||||
self.assertEqual(node.xpath('price/text()').getall(), [])
|
||||
|
||||
def test_xmliter_namespaced_nodename(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
|
||||
<channel>
|
||||
<title>My Dummy Company</title>
|
||||
<link>http://www.mydummycompany.com</link>
|
||||
<description>This is a dummy company. We do nothing.</description>
|
||||
<item>
|
||||
<title>Item 1</title>
|
||||
<description>This is item 1</description>
|
||||
<link>http://www.mydummycompany.com/items/1</link>
|
||||
<g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
|
||||
<g:id>ITEM_1</g:id>
|
||||
<g:price>400</g:price>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
response = XmlResponse(url='http://mydummycompany.com', body=body)
|
||||
my_iter = self.xmliter(response, 'g:image_link')
|
||||
node = next(my_iter)
|
||||
node.register_namespace('g', 'http://base.google.com/ns/1.0')
|
||||
self.assertEqual(node.xpath('text()').extract(), ['http://www.mydummycompany.com/images/item1.jpg'])
|
||||
|
||||
def test_xmliter_namespaced_nodename_missing(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0" xmlns:g="http://base.google.com/ns/1.0">
|
||||
<channel>
|
||||
<title>My Dummy Company</title>
|
||||
<link>http://www.mydummycompany.com</link>
|
||||
<description>This is a dummy company. We do nothing.</description>
|
||||
<item>
|
||||
<title>Item 1</title>
|
||||
<description>This is item 1</description>
|
||||
<link>http://www.mydummycompany.com/items/1</link>
|
||||
<g:image_link>http://www.mydummycompany.com/images/item1.jpg</g:image_link>
|
||||
<g:id>ITEM_1</g:id>
|
||||
<g:price>400</g:price>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
"""
|
||||
response = XmlResponse(url='http://mydummycompany.com', body=body)
|
||||
my_iter = self.xmliter(response, 'g:link_image')
|
||||
with self.assertRaises(StopIteration):
|
||||
next(my_iter)
|
||||
|
||||
def test_xmliter_exception(self):
|
||||
body = (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>'
|
||||
|
|
@ -183,6 +232,10 @@ class XmliterTestCase(unittest.TestCase):
|
|||
class LxmlXmliterTestCase(XmliterTestCase):
|
||||
xmliter = staticmethod(xmliter_lxml)
|
||||
|
||||
@mark.xfail(reason='known bug of the current implementation')
|
||||
def test_xmliter_namespaced_nodename(self):
|
||||
super().test_xmliter_namespaced_nodename()
|
||||
|
||||
def test_xmliter_iterate_namespace(self):
|
||||
body = b"""
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
|
|
|
|||
1
tox.ini
1
tox.ini
|
|
@ -16,7 +16,6 @@ deps =
|
|||
mitmproxy; python_version >= '3.7' and implementation_name != 'pypy'
|
||||
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
|
||||
# Extras
|
||||
boto3>=1.13.0
|
||||
botocore>=1.4.87
|
||||
Pillow>=4.0.0
|
||||
passenv =
|
||||
|
|
|
|||
Loading…
Reference in New Issue