mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into asyncio-startrequests-asyncgen
This commit is contained in:
Andrey Rakhmatullin
commit
33ebfcacca
|
|
@ -0,0 +1 @@
|
|||
tests/sample_data/** binary
|
||||
14
.travis.yml
14
.travis.yml
|
|
@ -19,16 +19,10 @@ matrix:
|
|||
python: 3.8
|
||||
|
||||
- env: TOXENV=pinned
|
||||
python: 3.5.2
|
||||
python: 3.6.1
|
||||
- env: TOXENV=asyncio-pinned
|
||||
python: 3.5.2 # We use additional code to support 3.5.3 and earlier
|
||||
- env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.5
|
||||
- env: TOXENV=asyncio
|
||||
python: 3.5 # We use specific code to support >= 3.5.4, < 3.6
|
||||
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
|
||||
python: 3.6.1
|
||||
- env: TOXENV=pypy3-pinned PYPY_VERSION=3.6-v7.2.0
|
||||
|
||||
- env: TOXENV=py
|
||||
python: 3.6
|
||||
|
|
@ -50,7 +44,7 @@ install:
|
|||
- |
|
||||
if [[ ! -z "$PYPY_VERSION" ]]; then
|
||||
export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
|
||||
wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"
|
||||
wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
|
||||
tar -jxf ${PYPY_VERSION}.tar.bz2
|
||||
virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"
|
||||
source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
|
||||
|
|
|
|||
25
appveyor.yml
25
appveyor.yml
|
|
@ -1,25 +0,0 @@
|
|||
platform: x86
|
||||
version: '{branch}-{build}'
|
||||
environment:
|
||||
matrix:
|
||||
- PYTHON: "C:\\Python36"
|
||||
TOX_ENV: py36
|
||||
|
||||
branches:
|
||||
only:
|
||||
- master
|
||||
- /d+\.\d+\.\d+[\w\-]*$/
|
||||
|
||||
install:
|
||||
- "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
|
||||
- "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%"
|
||||
- "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE"
|
||||
- "pip install -U tox"
|
||||
|
||||
build: false
|
||||
skip_tags: true
|
||||
test_script:
|
||||
- "tox -e %TOX_ENV%"
|
||||
|
||||
cache:
|
||||
- '%LOCALAPPDATA%\pip\cache'
|
||||
|
|
@ -4,11 +4,9 @@ pool:
|
|||
vmImage: 'windows-latest'
|
||||
strategy:
|
||||
matrix:
|
||||
Python35:
|
||||
python.version: '3.5'
|
||||
TOXENV: windows-pinned
|
||||
Python36:
|
||||
python.version: '3.6'
|
||||
TOXENV: windows-pinned
|
||||
Python37:
|
||||
python.version: '3.7'
|
||||
Python38:
|
||||
|
|
|
|||
|
|
@ -108,6 +108,11 @@ Well-written patches should:
|
|||
|
||||
tox -e docs-coverage
|
||||
|
||||
* if you are removing deprecated code, first make sure that at least 1 year
|
||||
(12 months) has passed since the release that introduced the deprecation.
|
||||
See :ref:`deprecation-policy`.
|
||||
|
||||
|
||||
.. _submitting-patches:
|
||||
|
||||
Submitting patches
|
||||
|
|
|
|||
|
|
@ -236,15 +236,15 @@ Simplest way to dump all my scraped items into a JSON/CSV/XML file?
|
|||
|
||||
To dump into a JSON file::
|
||||
|
||||
scrapy crawl myspider -o items.json
|
||||
scrapy crawl myspider -O items.json
|
||||
|
||||
To dump into a CSV file::
|
||||
|
||||
scrapy crawl myspider -o items.csv
|
||||
scrapy crawl myspider -O items.csv
|
||||
|
||||
To dump into a XML file::
|
||||
|
||||
scrapy crawl myspider -o items.xml
|
||||
scrapy crawl myspider -O items.xml
|
||||
|
||||
For more information see :ref:`topics-feed-exports`
|
||||
|
||||
|
|
|
|||
|
|
@ -42,30 +42,18 @@ http://quotes.toscrape.com, following the pagination::
|
|||
if next_page is not None:
|
||||
yield response.follow(next_page, self.parse)
|
||||
|
||||
|
||||
Put this in a text file, name it to something like ``quotes_spider.py``
|
||||
and run the spider using the :command:`runspider` command::
|
||||
|
||||
scrapy runspider quotes_spider.py -o quotes.json
|
||||
scrapy runspider quotes_spider.py -o quotes.jl
|
||||
|
||||
When this finishes you will have in the ``quotes.jl`` file a list of the
|
||||
quotes in JSON Lines format, containing text and author, looking like this::
|
||||
|
||||
When this finishes you will have in the ``quotes.json`` file a list of the
|
||||
quotes in JSON format, containing text and author, looking like this (reformatted
|
||||
here for better readability)::
|
||||
|
||||
[{
|
||||
"author": "Jane Austen",
|
||||
"text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"
|
||||
},
|
||||
{
|
||||
"author": "Groucho Marx",
|
||||
"text": "\u201cOutside of a dog, a book is man's best friend. Inside of a dog it's too dark to read.\u201d"
|
||||
},
|
||||
{
|
||||
"author": "Steve Martin",
|
||||
"text": "\u201cA day without sunshine is like, you know, night.\u201d"
|
||||
},
|
||||
...]
|
||||
{"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
|
||||
{"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
|
||||
{"author": "Garrison Keillor", "text": "\u201cAnyone who thinks sitting in church can make you a Christian must also think that sitting in a garage can make you a car.\u201d"}
|
||||
...
|
||||
|
||||
|
||||
What just happened?
|
||||
|
|
|
|||
|
|
@ -464,16 +464,15 @@ Storing the scraped data
|
|||
The simplest way to store the scraped data is by using :ref:`Feed exports
|
||||
<topics-feed-exports>`, with the following command::
|
||||
|
||||
scrapy crawl quotes -o quotes.json
|
||||
scrapy crawl quotes -O quotes.json
|
||||
|
||||
That will generate an ``quotes.json`` file containing all scraped items,
|
||||
serialized in `JSON`_.
|
||||
|
||||
For historic reasons, Scrapy appends to a given file instead of overwriting
|
||||
its contents. If you run this command twice without removing the file
|
||||
before the second time, you'll end up with a broken JSON file.
|
||||
|
||||
You can also use other formats, like `JSON Lines`_::
|
||||
The ``-O`` command-line switch overwrites any existing file; use ``-o`` instead
|
||||
to append new content to any existing file. However, appending to a JSON file
|
||||
makes the file contents invalid JSON. When appending to a file, consider
|
||||
using a different serialization format, such as `JSON Lines`_::
|
||||
|
||||
scrapy crawl quotes -o quotes.jl
|
||||
|
||||
|
|
@ -704,7 +703,7 @@ Using spider arguments
|
|||
You can provide command line arguments to your spiders by using the ``-a``
|
||||
option when running them::
|
||||
|
||||
scrapy crawl quotes -o quotes-humor.json -a tag=humor
|
||||
scrapy crawl quotes -O quotes-humor.json -a tag=humor
|
||||
|
||||
These arguments are passed to the Spider's ``__init__`` method and become
|
||||
spider attributes by default.
|
||||
|
|
|
|||
|
|
@ -26,3 +26,15 @@ reactor manually. You can do that using
|
|||
:func:`~scrapy.utils.reactor.install_reactor`::
|
||||
|
||||
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
|
||||
|
||||
.. _using-custom-loops:
|
||||
|
||||
Using custom asyncio loops
|
||||
==========================
|
||||
|
||||
You can also use custom asyncio event loops with the asyncio reactor. Set the
|
||||
:setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
|
||||
use it instead of the default asyncio event loop.
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -81,7 +81,7 @@ override three methods:
|
|||
.. class:: Contract(method, *args)
|
||||
|
||||
:param method: callback function to which the contract is associated
|
||||
:type method: function
|
||||
:type method: collections.abc.Callable
|
||||
|
||||
:param args: list of arguments passed into the docstring (whitespace
|
||||
separated)
|
||||
|
|
|
|||
|
|
@ -62,10 +62,10 @@ rest of the framework.
|
|||
:type smtpport: int
|
||||
|
||||
:param smtptls: enforce using SMTP STARTTLS
|
||||
:type smtptls: boolean
|
||||
:type smtptls: bool
|
||||
|
||||
:param smtpssl: enforce using a secure SSL connection
|
||||
:type smtpssl: boolean
|
||||
:type smtpssl: bool
|
||||
|
||||
.. classmethod:: from_settings(settings)
|
||||
|
||||
|
|
@ -79,14 +79,14 @@ rest of the framework.
|
|||
|
||||
Send email to the given recipients.
|
||||
|
||||
:param to: the e-mail recipients
|
||||
:type to: str or list of str
|
||||
:param to: the e-mail recipients as a string or as a list of strings
|
||||
:type to: str or list
|
||||
|
||||
:param subject: the subject of the e-mail
|
||||
:type subject: str
|
||||
|
||||
:param cc: the e-mails to CC
|
||||
:type cc: str or list of str
|
||||
:param cc: the e-mails to CC as a string or as a list of strings
|
||||
:type cc: str or list
|
||||
|
||||
:param body: the e-mail body
|
||||
:type body: str
|
||||
|
|
@ -96,7 +96,7 @@ rest of the framework.
|
|||
appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
|
||||
attachment and ``file_object`` is a readable file object with the
|
||||
contents of the attachment
|
||||
:type attachs: iterable
|
||||
:type attachs: collections.abc.Iterable
|
||||
|
||||
:param mimetype: the MIME type of the e-mail
|
||||
:type mimetype: str
|
||||
|
|
|
|||
|
|
@ -305,7 +305,7 @@ CsvItemExporter
|
|||
:param include_headers_line: If enabled, makes the exporter output a header
|
||||
line with the field names taken from
|
||||
:attr:`BaseItemExporter.fields_to_export` or the first exported item fields.
|
||||
:type include_headers_line: boolean
|
||||
:type include_headers_line: bool
|
||||
|
||||
:param join_multivalued: The char (or chars) that will be used for joining
|
||||
multi-valued fields, if found.
|
||||
|
|
|
|||
|
|
@ -291,6 +291,7 @@ Default: ``{}``
|
|||
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
|
||||
object) and each value is a nested dictionary containing configuration
|
||||
parameters for the specific feed.
|
||||
|
||||
This setting is required for enabling the feed export feature.
|
||||
|
||||
See :ref:`topics-feed-storage-backends` for supported URI schemes.
|
||||
|
|
@ -318,16 +319,43 @@ For instance::
|
|||
}
|
||||
|
||||
The following is a list of the accepted keys and the setting that is used
|
||||
as a fallback value if that key is not provided for a specific feed definition.
|
||||
as a fallback value if that key is not provided for a specific feed definition:
|
||||
|
||||
- ``format``: the :ref:`serialization format <topics-feed-format>`.
|
||||
|
||||
This setting is mandatory, there is no fallback value.
|
||||
|
||||
- ``batch_item_count``: falls back to
|
||||
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
|
||||
|
||||
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
|
||||
|
||||
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
|
||||
|
||||
- ``overwrite``: whether to overwrite the file if it already exists
|
||||
(``True``) or append to its content (``False``).
|
||||
|
||||
The default value depends on the :ref:`storage backend
|
||||
<topics-feed-storage-backends>`:
|
||||
|
||||
- :ref:`topics-feed-storage-fs`: ``False``
|
||||
|
||||
- :ref:`topics-feed-storage-ftp`: ``True``
|
||||
|
||||
.. note:: Some FTP servers may not support appending to files (the
|
||||
``APPE`` FTP command).
|
||||
|
||||
- :ref:`topics-feed-storage-s3`: ``True`` (appending `is not supported
|
||||
<https://forums.aws.amazon.com/message.jspa?messageID=540395>`_)
|
||||
|
||||
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
|
||||
|
||||
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
|
||||
|
||||
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
|
||||
|
||||
* ``format``: the serialization format to be used for the feed.
|
||||
See :ref:`topics-feed-format` for possible values.
|
||||
Mandatory, no fallback setting
|
||||
* ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`
|
||||
* ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`
|
||||
* ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`
|
||||
* ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`
|
||||
* ``batch_item_count``: falls back to :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`
|
||||
|
||||
.. setting:: FEED_EXPORT_ENCODING
|
||||
|
||||
|
|
@ -500,7 +528,7 @@ generated:
|
|||
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
|
||||
(e.g. ``2020-03-28T14-45-08.237134``)
|
||||
|
||||
* ``%(batch_id)d`` - gets replaced by the sequence number of the batch.
|
||||
* ``%(batch_id)d`` - gets replaced by the 1-based sequence number of the batch.
|
||||
|
||||
Use :ref:`printf-style string formatting <python:old-string-formatting>` to
|
||||
alter the number format. For example, to make the batch ID a 5-digit
|
||||
|
|
@ -517,16 +545,74 @@ And your :command:`crawl` command line is::
|
|||
|
||||
The command line above can generate a directory tree like::
|
||||
|
||||
->projectname
|
||||
-->dirname
|
||||
--->1-filename2020-03-28T14-45-08.237134.json
|
||||
--->2-filename2020-03-28T14-45-09.148903.json
|
||||
--->3-filename2020-03-28T14-45-10.046092.json
|
||||
->projectname
|
||||
-->dirname
|
||||
--->1-filename2020-03-28T14-45-08.237134.json
|
||||
--->2-filename2020-03-28T14-45-09.148903.json
|
||||
--->3-filename2020-03-28T14-45-10.046092.json
|
||||
|
||||
Where the first and second files contain exactly 100 items. The last one contains
|
||||
100 items or fewer.
|
||||
|
||||
|
||||
.. setting:: FEED_URI_PARAMS
|
||||
|
||||
FEED_URI_PARAMS
|
||||
---------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
A string with the import path of a function to set the parameters to apply with
|
||||
:ref:`printf-style string formatting <python:old-string-formatting>` to the
|
||||
feed URI.
|
||||
|
||||
The function signature should be as follows:
|
||||
|
||||
.. function:: uri_params(params, spider)
|
||||
|
||||
Return a :class:`dict` of key-value pairs to apply to the feed URI using
|
||||
:ref:`printf-style string formatting <python:old-string-formatting>`.
|
||||
|
||||
:param params: default key-value pairs
|
||||
|
||||
Specifically:
|
||||
|
||||
- ``batch_id``: ID of the file batch. See
|
||||
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
|
||||
is always ``1``.
|
||||
|
||||
- ``batch_time``: UTC date and time, in ISO format with ``:``
|
||||
replaced with ``-``.
|
||||
|
||||
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
|
||||
|
||||
- ``time``: ``batch_time``, with microseconds set to ``0``.
|
||||
:type params: dict
|
||||
|
||||
:param spider: source spider of the feed items
|
||||
:type spider: scrapy.spiders.Spider
|
||||
|
||||
For example, to include the :attr:`name <scrapy.spiders.Spider.name>` of the
|
||||
source spider in the feed URI:
|
||||
|
||||
#. Define the following function somewhere in your project::
|
||||
|
||||
# myproject/utils.py
|
||||
def uri_params(params, spider):
|
||||
return {**params, 'spider_name': spider.name}
|
||||
|
||||
#. Point :setting:`FEED_URI_PARAMS` to that function in your settings::
|
||||
|
||||
# myproject/settings.py
|
||||
FEED_URI_PARAMS = 'myproject.utils.uri_params'
|
||||
|
||||
#. Use ``%(spider_name)s`` in your feed URI::
|
||||
|
||||
scrapy crawl <spider_name> -o "%(spider_name)s.jl"
|
||||
|
||||
|
||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||
.. _botocore: https://github.com/boto/botocore
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
|||
|
||||
:param ignore: if given, all objects from the specified class (or tuple of
|
||||
classes) will be ignored.
|
||||
:type ignore: class or classes tuple
|
||||
:type ignore: type or tuple
|
||||
|
||||
.. function:: get_oldest(class_name)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,13 +46,13 @@ LxmlLinkExtractor
|
|||
:param allow: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be extracted. If not
|
||||
given (or empty), it will match all links.
|
||||
:type allow: a regular expression (or list of)
|
||||
:type allow: str or list
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: a regular expression (or list of)
|
||||
:type deny: str or list
|
||||
|
||||
:param allow_domains: a single value or a list of string containing
|
||||
domains which will be considered for extracting the links
|
||||
|
|
@ -88,7 +88,7 @@ LxmlLinkExtractor
|
|||
that the link's text must match in order to be extracted. If not
|
||||
given (or empty), it will match all links. If a list of regular expressions is
|
||||
given, the link will be extracted if it matches at least one.
|
||||
:type restrict_text: a regular expression (or list of)
|
||||
:type restrict_text: str or list
|
||||
|
||||
:param tags: a tag or a list of tags to consider when extracting links.
|
||||
Defaults to ``('a', 'area')``.
|
||||
|
|
@ -106,11 +106,11 @@ LxmlLinkExtractor
|
|||
different for requests with canonicalized and raw URLs. If you're
|
||||
using LinkExtractor to follow links it is more robust to
|
||||
keep the default ``canonicalize=False``.
|
||||
:type canonicalize: boolean
|
||||
:type canonicalize: bool
|
||||
|
||||
:param unique: whether duplicate filtering should be applied to extracted
|
||||
links.
|
||||
:type unique: boolean
|
||||
:type unique: bool
|
||||
|
||||
:param process_value: a function which receives each value extracted from
|
||||
the tag and attributes scanned and can modify the value and return a
|
||||
|
|
@ -132,7 +132,7 @@ LxmlLinkExtractor
|
|||
if m:
|
||||
return m.group(1)
|
||||
|
||||
:type process_value: callable
|
||||
:type process_value: collections.abc.Callable
|
||||
|
||||
:param strip: whether to strip whitespaces from extracted attributes.
|
||||
According to HTML5 standard, leading and trailing whitespaces
|
||||
|
|
@ -141,7 +141,7 @@ LxmlLinkExtractor
|
|||
elements, etc., so LinkExtractor strips space chars by default.
|
||||
Set ``strip=False`` to turn it off (e.g. if you're extracting urls
|
||||
from elements or attributes which allow leading/trailing whitespaces).
|
||||
:type strip: boolean
|
||||
:type strip: bool
|
||||
|
||||
.. automethod:: extract_links
|
||||
|
||||
|
|
|
|||
|
|
@ -412,15 +412,16 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
.. class:: FilesPipeline
|
||||
|
||||
.. method:: file_path(self, request, response=None, info=None)
|
||||
.. method:: file_path(self, request, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called once per downloaded item. It returns the
|
||||
download path of the file originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>` and
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
|
||||
:class:`request <scrapy.Request>`,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.item.Item>`
|
||||
|
||||
You can override this method to customize the download path of each file.
|
||||
|
||||
|
|
@ -436,9 +437,12 @@ See here the methods that you can override in your custom Files Pipeline:
|
|||
|
||||
class MyFilesPipeline(FilesPipeline):
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return 'files/' + os.path.basename(urlparse(request.url).path)
|
||||
|
||||
Similarly, you can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`file_path` method returns
|
||||
``full/<request URL hash>.<extension>``.
|
||||
|
||||
|
|
@ -544,15 +548,16 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
The :class:`ImagesPipeline` is an extension of the :class:`FilesPipeline`,
|
||||
customizing the field names and adding custom behavior for images.
|
||||
|
||||
.. method:: file_path(self, request, response=None, info=None)
|
||||
.. method:: file_path(self, request, response=None, info=None, *, item=None)
|
||||
|
||||
This method is called once per downloaded item. It returns the
|
||||
download path of the file originating from the specified
|
||||
:class:`response <scrapy.http.Response>`.
|
||||
|
||||
In addition to ``response``, this method receives the original
|
||||
:class:`request <scrapy.Request>` and
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>`.
|
||||
:class:`request <scrapy.Request>`,
|
||||
:class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
|
||||
:class:`item <scrapy.item.Item>`
|
||||
|
||||
You can override this method to customize the download path of each file.
|
||||
|
||||
|
|
@ -568,9 +573,12 @@ See here the methods that you can override in your custom Images Pipeline:
|
|||
|
||||
class MyImagesPipeline(ImagesPipeline):
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
return 'files/' + os.path.basename(urlparse(request.url).path)
|
||||
|
||||
Similarly, you can use the ``item`` to determine the file path based on some item
|
||||
property.
|
||||
|
||||
By default the :meth:`file_path` method returns
|
||||
``full/<request URL hash>.<extension>``.
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ Request objects
|
|||
:param url: the URL of this request
|
||||
|
||||
If the URL is invalid, a :exc:`ValueError` exception is raised.
|
||||
:type url: string
|
||||
:type url: str
|
||||
|
||||
:param callback: the function that will be called with the response of this
|
||||
request (once it's downloaded) as its first parameter. For more information
|
||||
|
|
@ -42,10 +42,10 @@ Request objects
|
|||
:meth:`~scrapy.spiders.Spider.parse` method will be used.
|
||||
Note that if exceptions are raised during processing, errback is called instead.
|
||||
|
||||
:type callback: callable
|
||||
:type callback: collections.abc.Callable
|
||||
|
||||
:param method: the HTTP method of this request. Defaults to ``'GET'``.
|
||||
:type method: string
|
||||
:type method: str
|
||||
|
||||
:param meta: the initial values for the :attr:`Request.meta` attribute. If
|
||||
given, the dict passed in this parameter will be shallow copied.
|
||||
|
|
@ -107,7 +107,7 @@ Request objects
|
|||
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
|
||||
This encoding will be used to percent-encode the URL and to convert the
|
||||
body to bytes (if given as a string).
|
||||
:type encoding: string
|
||||
:type encoding: str
|
||||
|
||||
:param priority: the priority of this request (defaults to ``0``).
|
||||
The priority is used by the scheduler to define the order used to process
|
||||
|
|
@ -119,7 +119,7 @@ Request objects
|
|||
the scheduler. This is used when you want to perform an identical
|
||||
request multiple times, to ignore the duplicates filter. Use it with
|
||||
care, or you will get into crawling loops. Default to ``False``.
|
||||
:type dont_filter: boolean
|
||||
:type dont_filter: bool
|
||||
|
||||
:param errback: a function that will be called if any exception was
|
||||
raised while processing the request. This includes pages that failed
|
||||
|
|
@ -131,7 +131,7 @@ Request objects
|
|||
.. versionchanged:: 2.0
|
||||
The *callback* parameter is no longer required when the *errback*
|
||||
parameter is specified.
|
||||
:type errback: callable
|
||||
:type errback: collections.abc.Callable
|
||||
|
||||
:param flags: Flags sent to the request, can be used for logging or similar purposes.
|
||||
:type flags: list
|
||||
|
|
@ -159,7 +159,7 @@ Request objects
|
|||
|
||||
.. attribute:: Request.body
|
||||
|
||||
A str that contains the request body.
|
||||
The request body as bytes.
|
||||
|
||||
This attribute is read-only. To change the body of a Request use
|
||||
:meth:`replace`.
|
||||
|
|
@ -485,7 +485,7 @@ fields with form data from :class:`Response` objects.
|
|||
:param formdata: is a dictionary (or iterable of (key, value) tuples)
|
||||
containing HTML Form data which will be url-encoded and assigned to the
|
||||
body of the request.
|
||||
:type formdata: dict or iterable of tuples
|
||||
:type formdata: dict or collections.abc.Iterable
|
||||
|
||||
The :class:`FormRequest` objects support the following class method in
|
||||
addition to the standard :class:`Request` methods:
|
||||
|
|
@ -517,20 +517,20 @@ fields with form data from :class:`Response` objects.
|
|||
:type response: :class:`Response` object
|
||||
|
||||
:param formname: if given, the form with name attribute set to this value will be used.
|
||||
:type formname: string
|
||||
:type formname: str
|
||||
|
||||
:param formid: if given, the form with id attribute set to this value will be used.
|
||||
:type formid: string
|
||||
:type formid: str
|
||||
|
||||
:param formxpath: if given, the first form that matches the xpath will be used.
|
||||
:type formxpath: string
|
||||
:type formxpath: str
|
||||
|
||||
:param formcss: if given, the first form that matches the css selector will be used.
|
||||
:type formcss: string
|
||||
:type formcss: str
|
||||
|
||||
:param formnumber: the number of form to use, when the response contains
|
||||
multiple forms. The first one (and also the default) is ``0``.
|
||||
:type formnumber: integer
|
||||
:type formnumber: int
|
||||
|
||||
:param formdata: fields to override in the form data. If a field was
|
||||
already present in the response ``<form>`` element, its value is
|
||||
|
|
@ -548,7 +548,7 @@ fields with form data from :class:`Response` objects.
|
|||
|
||||
:param dont_click: If True, the form data will be submitted without
|
||||
clicking in any element.
|
||||
:type dont_click: boolean
|
||||
:type dont_click: bool
|
||||
|
||||
The other parameters of this class method are passed directly to the
|
||||
:class:`FormRequest` ``__init__`` method.
|
||||
|
|
@ -636,7 +636,7 @@ dealing with JSON requests.
|
|||
if :attr:`Request.body` argument is provided this parameter will be ignored.
|
||||
if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
|
||||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
:type data: object
|
||||
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
|
||||
data into JSON format.
|
||||
|
|
@ -663,16 +663,16 @@ Response objects
|
|||
downloaded (by the Downloader) and fed to the Spiders for processing.
|
||||
|
||||
:param url: the URL of this response
|
||||
:type url: string
|
||||
:type url: str
|
||||
|
||||
:param status: the HTTP status of the response. Defaults to ``200``.
|
||||
:type status: integer
|
||||
:type status: int
|
||||
|
||||
:param headers: the headers of this response. The dict values can be strings
|
||||
(for single valued headers) or lists (for multi-valued headers).
|
||||
:type headers: dict
|
||||
|
||||
:param body: the response body. To access the decoded text as str you can use
|
||||
:param body: the response body. To access the decoded text as a string, use
|
||||
``response.text`` from an encoding-aware
|
||||
:ref:`Response subclass <topics-request-response-ref-response-subclasses>`,
|
||||
such as :class:`TextResponse`.
|
||||
|
|
@ -720,10 +720,10 @@ Response objects
|
|||
|
||||
.. attribute:: Response.body
|
||||
|
||||
The body of this Response. Keep in mind that Response.body
|
||||
is always a bytes object. If you want the string version use
|
||||
:attr:`TextResponse.text` (only available in :class:`TextResponse`
|
||||
and subclasses).
|
||||
The response body as bytes.
|
||||
|
||||
If you want the body as a string, use :attr:`TextResponse.text` (only
|
||||
available in :class:`TextResponse` and subclasses).
|
||||
|
||||
This attribute is read-only. To change the body of a Response use
|
||||
:meth:`replace`.
|
||||
|
|
@ -843,10 +843,10 @@ TextResponse objects
|
|||
|
||||
:param encoding: is a string which contains the encoding to use for this
|
||||
response. If you create a :class:`TextResponse` object with a string as
|
||||
body, it will be encoded using this encoding (remember the body attribute
|
||||
is always a bytes object). If ``encoding`` is ``None`` (default value), the
|
||||
encoding will be looked up in the response headers and body instead.
|
||||
:type encoding: string
|
||||
body, it will be converted to bytes encoded using this encoding. If
|
||||
*encoding* is ``None`` (default), the encoding will be looked up in the
|
||||
response headers and body instead.
|
||||
:type encoding: str
|
||||
|
||||
:class:`TextResponse` objects support the following attributes in addition
|
||||
to the standard :class:`Response` ones:
|
||||
|
|
|
|||
|
|
@ -98,6 +98,32 @@ class.
|
|||
The global defaults are located in the ``scrapy.settings.default_settings``
|
||||
module and documented in the :ref:`topics-settings-ref` section.
|
||||
|
||||
|
||||
Import paths and classes
|
||||
========================
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
When a setting references a callable object to be imported by Scrapy, such as a
|
||||
class or a function, there are two different ways you can specify that object:
|
||||
|
||||
- As a string containing the import path of that object
|
||||
|
||||
- As the object itself
|
||||
|
||||
For example::
|
||||
|
||||
from mybot.pipelines.validate import ValidateMyItem
|
||||
ITEM_PIPELINES = {
|
||||
# passing the classname...
|
||||
ValidateMyItem: 300,
|
||||
# ...equals passing the class path
|
||||
'mybot.pipelines.validate.ValidateMyItem': 300,
|
||||
}
|
||||
|
||||
.. note:: Passing non-callable objects is not supported.
|
||||
|
||||
|
||||
How to access settings
|
||||
======================
|
||||
|
||||
|
|
@ -216,6 +242,26 @@ Default: ``None``
|
|||
|
||||
The name of the region associated with the AWS client.
|
||||
|
||||
.. setting:: ASYNCIO_EVENT_LOOP
|
||||
|
||||
ASYNCIO_EVENT_LOOP
|
||||
------------------
|
||||
|
||||
Default: ``None``
|
||||
|
||||
Import path of a given asyncio event loop class.
|
||||
|
||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
|
||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||
event loop will be used.
|
||||
|
||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||
class to be used.
|
||||
|
||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||
|
||||
.. setting:: BOT_NAME
|
||||
|
||||
BOT_NAME
|
||||
|
|
|
|||
|
|
@ -423,6 +423,11 @@ response_received
|
|||
:param spider: the spider for which the response is intended
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. note:: The ``request`` argument might not contain the original request that
|
||||
reached the downloader, if a :ref:`topics-downloader-middleware` modifies
|
||||
the :class:`~scrapy.http.Response` object and sets a specific ``request``
|
||||
attribute.
|
||||
|
||||
response_downloaded
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
.. _versioning:
|
||||
|
||||
============================
|
||||
Versioning and API Stability
|
||||
Versioning and API stability
|
||||
============================
|
||||
|
||||
Versioning
|
||||
|
|
@ -34,7 +34,7 @@ For example:
|
|||
production)
|
||||
|
||||
|
||||
API Stability
|
||||
API stability
|
||||
=============
|
||||
|
||||
API stability was one of the major goals for the *1.0* release.
|
||||
|
|
@ -47,5 +47,23 @@ new methods or functionality but the existing methods should keep working the
|
|||
same way.
|
||||
|
||||
|
||||
.. _deprecation-policy:
|
||||
|
||||
Deprecation policy
|
||||
==================
|
||||
|
||||
We aim to maintain support for deprecated Scrapy features for at least 1 year.
|
||||
|
||||
For example, if a feature is deprecated in a Scrapy version released on
|
||||
June 15th 2020, that feature should continue to work in versions released on
|
||||
June 14th 2021 or before that.
|
||||
|
||||
Any new Scrapy release after a year *may* remove support for that deprecated
|
||||
feature.
|
||||
|
||||
All deprecated features removed in a Scrapy release are explicitly mentioned in
|
||||
the :ref:`release notes <news>`.
|
||||
|
||||
|
||||
.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
|
||||
|
||||
|
|
|
|||
2
pylintrc
2
pylintrc
|
|
@ -68,6 +68,7 @@ disable=abstract-method,
|
|||
pointless-statement,
|
||||
pointless-string-statement,
|
||||
protected-access,
|
||||
raise-missing-from,
|
||||
redefined-argument-from-local,
|
||||
redefined-builtin,
|
||||
redefined-outer-name,
|
||||
|
|
@ -75,6 +76,7 @@ disable=abstract-method,
|
|||
signature-differs,
|
||||
singleton-comparison,
|
||||
super-init-not-called,
|
||||
super-with-arguments,
|
||||
superfluous-parens,
|
||||
too-few-public-methods,
|
||||
too-many-ancestors,
|
||||
|
|
|
|||
|
|
@ -115,9 +115,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
parser.add_option("-a", dest="spargs", action="append", default=[], metavar="NAME=VALUE",
|
||||
help="set spider argument (may be repeated)")
|
||||
parser.add_option("-o", "--output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE (use - for stdout)")
|
||||
help="append scraped items to the end of FILE (use - for stdout)")
|
||||
parser.add_option("-O", "--overwrite-output", metavar="FILE", action="append",
|
||||
help="dump scraped items into FILE, overwriting any existing file")
|
||||
parser.add_option("-t", "--output-format", metavar="FORMAT",
|
||||
help="format to use for dumping items with -o")
|
||||
help="format to use for dumping items")
|
||||
|
||||
def process_options(self, args, opts):
|
||||
ScrapyCommand.process_options(self, args, opts)
|
||||
|
|
@ -125,6 +127,11 @@ class BaseRunSpiderCommand(ScrapyCommand):
|
|||
opts.spargs = arglist_to_dict(opts.spargs)
|
||||
except ValueError:
|
||||
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
|
||||
if opts.output:
|
||||
feeds = feed_process_params_from_cli(self.settings, opts.output, opts.output_format)
|
||||
if opts.output or opts.overwrite_output:
|
||||
feeds = feed_process_params_from_cli(
|
||||
self.settings,
|
||||
opts.output,
|
||||
opts.output_format,
|
||||
opts.overwrite_output,
|
||||
)
|
||||
self.settings.set('FEEDS', feeds, priority='cmdline')
|
||||
|
|
|
|||
|
|
@ -66,16 +66,9 @@ class Command(ScrapyCommand):
|
|||
print("Cannot create a spider with the same name as your project")
|
||||
return
|
||||
|
||||
try:
|
||||
spidercls = self.crawler_process.spider_loader.load(name)
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
# if spider already exists and not --force then halt
|
||||
if not opts.force:
|
||||
print("Spider %r already exists in module:" % name)
|
||||
print(" %s" % spidercls.__module__)
|
||||
return
|
||||
if not opts.force and self._spider_exists(name):
|
||||
return
|
||||
|
||||
template_file = self._find_template(opts.template)
|
||||
if template_file:
|
||||
self._genspider(module, name, domain, opts.template, template_file)
|
||||
|
|
@ -119,6 +112,34 @@ class Command(ScrapyCommand):
|
|||
if filename.endswith('.tmpl'):
|
||||
print(" %s" % splitext(filename)[0])
|
||||
|
||||
def _spider_exists(self, name):
|
||||
if not self.settings.get('NEWSPIDER_MODULE'):
|
||||
# if run as a standalone command and file with same filename already exists
|
||||
if exists(name + ".py"):
|
||||
print("%s already exists" % (abspath(name + ".py")))
|
||||
return True
|
||||
return False
|
||||
|
||||
try:
|
||||
spidercls = self.crawler_process.spider_loader.load(name)
|
||||
except KeyError:
|
||||
pass
|
||||
else:
|
||||
# if spider with same name exists
|
||||
print("Spider %r already exists in module:" % name)
|
||||
print(" %s" % spidercls.__module__)
|
||||
return True
|
||||
|
||||
# a file with the same name exists in the target directory
|
||||
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
|
||||
spiders_dir = dirname(spiders_module.__file__)
|
||||
spiders_dir_abs = abspath(spiders_dir)
|
||||
if exists(join(spiders_dir_abs, name + ".py")):
|
||||
print("%s already exists" % (join(spiders_dir_abs, (name + ".py"))))
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
@property
|
||||
def templates_dir(self):
|
||||
return join(
|
||||
|
|
|
|||
|
|
@ -394,13 +394,14 @@ class ScrapyAgent:
|
|||
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
|
||||
|
||||
if maxsize and expected_size > maxsize:
|
||||
error_msg = ("Cancelling download of %(url)s: expected response "
|
||||
"size (%(size)s) larger than download max size (%(maxsize)s).")
|
||||
error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
|
||||
warning_msg = ("Cancelling download of %(url)s: expected response "
|
||||
"size (%(size)s) larger than download max size (%(maxsize)s).")
|
||||
warning_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
|
||||
|
||||
logger.warning(warning_msg, warning_args)
|
||||
|
||||
logger.error(error_msg, error_args)
|
||||
txresponse._transport._producer.loseConnection()
|
||||
raise defer.CancelledError(error_msg % error_args)
|
||||
raise defer.CancelledError(warning_msg % warning_args)
|
||||
|
||||
if warnsize and expected_size > warnsize:
|
||||
logger.warning("Expected response size (%(size)s) larger than "
|
||||
|
|
@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol):
|
|||
self._finish_response(flags=["download_stopped"], failure=failure)
|
||||
|
||||
if self._maxsize and self._bytes_received > self._maxsize:
|
||||
logger.error("Received (%(bytes)s) bytes larger than download "
|
||||
"max size (%(maxsize)s) in request %(request)s.",
|
||||
{'bytes': self._bytes_received,
|
||||
'maxsize': self._maxsize,
|
||||
'request': self._request})
|
||||
logger.warning("Received (%(bytes)s) bytes larger than download "
|
||||
"max size (%(maxsize)s) in request %(request)s.",
|
||||
{'bytes': self._bytes_received,
|
||||
'maxsize': self._maxsize,
|
||||
'request': self._request})
|
||||
# Clear buffer earlier to avoid keeping data in memory for a long time.
|
||||
self._bodybuf.truncate(0)
|
||||
self._finished.cancel()
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
from time import time
|
||||
from urllib.parse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.web.client import HTTPClientFactory
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer
|
||||
from twisted.internet import defer, reactor
|
||||
from twisted.internet.protocol import ClientFactory
|
||||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
|
|
@ -92,18 +92,34 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
% (self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
"""Scrapy implementation of the HTTPClientFactory overwriting the
|
||||
setUrl method to make use of our Url object that cache the parse
|
||||
result.
|
||||
"""
|
||||
# This class used to inherit from Twisted’s
|
||||
# twisted.web.client.HTTPClientFactory. When that class was deprecated in
|
||||
# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
|
||||
# non-overriden code into this class.
|
||||
class ScrapyHTTPClientFactory(ClientFactory):
|
||||
|
||||
protocol = ScrapyHTTPPageGetter
|
||||
|
||||
waiting = 1
|
||||
noisy = False
|
||||
followRedirect = False
|
||||
afterFoundGet = False
|
||||
|
||||
def _build_response(self, body, request):
|
||||
request.meta['download_latency'] = self.headers_time - self.start_time
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body)
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
parsed = urlparse_cached(request)
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
|
||||
proxy = request.meta.get('proxy')
|
||||
if proxy:
|
||||
self.scheme, _, self.host, self.port, _ = _parse(proxy)
|
||||
self.path = self.url
|
||||
|
||||
def __init__(self, request, timeout=180):
|
||||
self._url = urldefrag(request.url)[0]
|
||||
# converting to bytes to comply to Twisted interface
|
||||
|
|
@ -138,21 +154,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
elif self.method == b'POST':
|
||||
self.headers['Content-Length'] = 0
|
||||
|
||||
def _build_response(self, body, request):
|
||||
request.meta['download_latency'] = self.headers_time - self.start_time
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
return respcls(url=self._url, status=status, headers=headers, body=body)
|
||||
def __repr__(self):
|
||||
return "<%s: %s>" % (self.__class__.__name__, self.url)
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
parsed = urlparse_cached(request)
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
|
||||
proxy = request.meta.get('proxy')
|
||||
if proxy:
|
||||
self.scheme, _, self.host, self.port, _ = _parse(proxy)
|
||||
self.path = self.url
|
||||
def _cancelTimeout(self, result, timeoutCall):
|
||||
if timeoutCall.active():
|
||||
timeoutCall.cancel()
|
||||
return result
|
||||
|
||||
def buildProtocol(self, addr):
|
||||
p = ClientFactory.buildProtocol(self, addr)
|
||||
p.followRedirect = self.followRedirect
|
||||
p.afterFoundGet = self.afterFoundGet
|
||||
if self.timeout:
|
||||
timeoutCall = reactor.callLater(self.timeout, p.timeout)
|
||||
self.deferred.addBoth(self._cancelTimeout, timeoutCall)
|
||||
return p
|
||||
|
||||
def gotHeaders(self, headers):
|
||||
self.headers_time = time()
|
||||
self.response_headers = headers
|
||||
|
||||
def gotStatus(self, version, status, message):
|
||||
"""
|
||||
Set the status of the request on us.
|
||||
@param version: The HTTP version.
|
||||
@type version: L{bytes}
|
||||
@param status: The HTTP status code, an integer represented as a
|
||||
bytestring.
|
||||
@type status: L{bytes}
|
||||
@param message: The HTTP status message.
|
||||
@type message: L{bytes}
|
||||
"""
|
||||
self.version, self.status, self.message = version, status, message
|
||||
|
||||
def page(self, page):
|
||||
if self.waiting:
|
||||
self.waiting = 0
|
||||
self.deferred.callback(page)
|
||||
|
||||
def noPage(self, reason):
|
||||
if self.waiting:
|
||||
self.waiting = 0
|
||||
self.deferred.errback(reason)
|
||||
|
||||
def clientConnectionFailed(self, _, reason):
|
||||
"""
|
||||
When a connection attempt fails, the request cannot be issued. If no
|
||||
result has yet been provided to the result Deferred, provide the
|
||||
connection failure reason as an error result.
|
||||
"""
|
||||
if self.waiting:
|
||||
self.waiting = 0
|
||||
# If the connection attempt failed, there is nothing more to
|
||||
# disconnect, so just fire that Deferred now.
|
||||
self._disconnectedDeferred.callback(None)
|
||||
self.deferred.errback(reason)
|
||||
|
|
|
|||
|
|
@ -271,12 +271,17 @@ class ExecutionEngine:
|
|||
% (type(response), response)
|
||||
)
|
||||
if isinstance(response, Response):
|
||||
response.request = request # tie request to response received
|
||||
logkws = self.logformatter.crawled(request, response, spider)
|
||||
if response.request is None:
|
||||
response.request = request
|
||||
logkws = self.logformatter.crawled(response.request, response, spider)
|
||||
if logkws is not None:
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider})
|
||||
self.signals.send_catch_log(signals.response_received,
|
||||
response=response, request=request, spider=spider)
|
||||
self.signals.send_catch_log(
|
||||
signal=signals.response_received,
|
||||
response=response,
|
||||
request=response.request,
|
||||
spider=spider,
|
||||
)
|
||||
return response
|
||||
|
||||
def _on_complete(_):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from scrapy import signals
|
|||
from scrapy.core.spidermw import SpiderMiddlewareManager
|
||||
from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.utils.defer import defer_result, defer_succeed, iter_errback, parallel
|
||||
from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
|
||||
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
|
||||
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
|
||||
from scrapy.utils.spider import iterate_spider_output
|
||||
|
|
@ -120,40 +120,40 @@ class Scraper:
|
|||
response, request, deferred = slot.next_response_request_deferred()
|
||||
self._scrape(response, request, spider).chainDeferred(deferred)
|
||||
|
||||
def _scrape(self, response, request, spider):
|
||||
"""Handle the downloaded response or failure through the spider
|
||||
callback/errback"""
|
||||
if not isinstance(response, (Response, Failure)):
|
||||
raise TypeError(
|
||||
"Incorrect type: expected Response or Failure, got %s: %r"
|
||||
% (type(response), response)
|
||||
)
|
||||
|
||||
dfd = self._scrape2(response, request, spider) # returns spider's processed output
|
||||
dfd.addErrback(self.handle_spider_error, request, response, spider)
|
||||
dfd.addCallback(self.handle_spider_output, request, response, spider)
|
||||
def _scrape(self, result, request, spider):
|
||||
"""
|
||||
Handle the downloaded response or failure through the spider callback/errback
|
||||
"""
|
||||
if not isinstance(result, (Response, Failure)):
|
||||
raise TypeError("Incorrect type: expected Response or Failure, got %s: %r" % (type(result), result))
|
||||
dfd = self._scrape2(result, request, spider) # returns spider's processed output
|
||||
dfd.addErrback(self.handle_spider_error, request, result, spider)
|
||||
dfd.addCallback(self.handle_spider_output, request, result, spider)
|
||||
return dfd
|
||||
|
||||
def _scrape2(self, request_result, request, spider):
|
||||
"""Handle the different cases of request's result been a Response or a
|
||||
Failure"""
|
||||
if not isinstance(request_result, Failure):
|
||||
return self.spidermw.scrape_response(
|
||||
self.call_spider, request_result, request, spider)
|
||||
else:
|
||||
dfd = self.call_spider(request_result, request, spider)
|
||||
return dfd.addErrback(
|
||||
self._log_download_errors, request_result, request, spider)
|
||||
def _scrape2(self, result, request, spider):
|
||||
"""
|
||||
Handle the different cases of request's result been a Response or a Failure
|
||||
"""
|
||||
if isinstance(result, Response):
|
||||
return self.spidermw.scrape_response(self.call_spider, result, request, spider)
|
||||
else: # result is a Failure
|
||||
dfd = self.call_spider(result, request, spider)
|
||||
return dfd.addErrback(self._log_download_errors, result, request, spider)
|
||||
|
||||
def call_spider(self, result, request, spider):
|
||||
result.request = request
|
||||
dfd = defer_result(result)
|
||||
callback = request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
dfd.addCallbacks(callback=callback,
|
||||
errback=request.errback,
|
||||
callbackKeywords=request.cb_kwargs)
|
||||
if isinstance(result, Response):
|
||||
if getattr(result, "request", None) is None:
|
||||
result.request = request
|
||||
callback = result.request.callback or spider._parse
|
||||
warn_on_generator_with_return_value(spider, callback)
|
||||
dfd = defer_succeed(result)
|
||||
dfd.addCallback(callback, **result.request.cb_kwargs)
|
||||
else: # result is a Failure
|
||||
result.request = request
|
||||
warn_on_generator_with_return_value(spider, request.errback)
|
||||
dfd = defer_fail(result)
|
||||
dfd.addErrback(request.errback)
|
||||
return dfd.addCallback(iterate_spider_output)
|
||||
|
||||
def handle_spider_error(self, _failure, request, response, spider):
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@ class CrawlerProcess(CrawlerRunner):
|
|||
If ``stop_after_crawl`` is True, the reactor will be stopped after all
|
||||
crawlers have finished, using :meth:`join`.
|
||||
|
||||
:param boolean stop_after_crawl: stop or not the reactor when all
|
||||
:param bool stop_after_crawl: stop or not the reactor when all
|
||||
crawlers have finished
|
||||
"""
|
||||
from twisted.internet import reactor
|
||||
|
|
@ -359,5 +359,5 @@ class CrawlerProcess(CrawlerRunner):
|
|||
|
||||
def _handle_twisted_reactor(self):
|
||||
if self.settings.get("TWISTED_REACTOR"):
|
||||
install_reactor(self.settings["TWISTED_REACTOR"])
|
||||
install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
|
||||
super()._handle_twisted_reactor()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
from email.utils import formatdate
|
||||
from typing import Optional, Type, TypeVar
|
||||
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.error import (
|
||||
|
|
@ -13,10 +14,19 @@ from twisted.internet.error import (
|
|||
from twisted.web.client import ResponseFailed
|
||||
|
||||
from scrapy import signals
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||
from scrapy.http.request import Request
|
||||
from scrapy.http.response import Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
HttpCacheMiddlewareTV = TypeVar("HttpCacheMiddlewareTV", bound="HttpCacheMiddleware")
|
||||
|
||||
|
||||
class HttpCacheMiddleware:
|
||||
|
||||
DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError,
|
||||
|
|
@ -24,7 +34,7 @@ class HttpCacheMiddleware:
|
|||
ConnectionLost, TCPTimedOutError, ResponseFailed,
|
||||
IOError)
|
||||
|
||||
def __init__(self, settings, stats):
|
||||
def __init__(self, settings: Settings, stats: StatsCollector) -> None:
|
||||
if not settings.getbool('HTTPCACHE_ENABLED'):
|
||||
raise NotConfigured
|
||||
self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings)
|
||||
|
|
@ -33,26 +43,26 @@ class HttpCacheMiddleware:
|
|||
self.stats = stats
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
def from_crawler(cls: Type[HttpCacheMiddlewareTV], crawler: Crawler) -> HttpCacheMiddlewareTV:
|
||||
o = cls(crawler.settings, crawler.stats)
|
||||
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
|
||||
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
|
||||
return o
|
||||
|
||||
def spider_opened(self, spider):
|
||||
def spider_opened(self, spider: Spider) -> None:
|
||||
self.storage.open_spider(spider)
|
||||
|
||||
def spider_closed(self, spider):
|
||||
def spider_closed(self, spider: Spider) -> None:
|
||||
self.storage.close_spider(spider)
|
||||
|
||||
def process_request(self, request, spider):
|
||||
def process_request(self, request: Request, spider: Spider) -> Optional[Response]:
|
||||
if request.meta.get('dont_cache', False):
|
||||
return
|
||||
return None
|
||||
|
||||
# Skip uncacheable requests
|
||||
if not self.policy.should_cache_request(request):
|
||||
request.meta['_dont_cache'] = True # flag as uncacheable
|
||||
return
|
||||
return None
|
||||
|
||||
# Look for cached response and check if expired
|
||||
cachedresponse = self.storage.retrieve_response(spider, request)
|
||||
|
|
@ -61,7 +71,7 @@ class HttpCacheMiddleware:
|
|||
if self.ignore_missing:
|
||||
self.stats.inc_value('httpcache/ignore', spider=spider)
|
||||
raise IgnoreRequest("Ignored request not in cache: %s" % request)
|
||||
return # first time request
|
||||
return None # first time request
|
||||
|
||||
# Return cached response only if not expired
|
||||
cachedresponse.flags.append('cached')
|
||||
|
|
@ -73,7 +83,9 @@ class HttpCacheMiddleware:
|
|||
# process_response hook
|
||||
request.meta['cached_response'] = cachedresponse
|
||||
|
||||
def process_response(self, request, response, spider):
|
||||
return None
|
||||
|
||||
def process_response(self, request: Request, response: Response, spider: Spider) -> Response:
|
||||
if request.meta.get('dont_cache', False):
|
||||
return response
|
||||
|
||||
|
|
@ -85,7 +97,7 @@ class HttpCacheMiddleware:
|
|||
# RFC2616 requires origin server to set Date header,
|
||||
# https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18
|
||||
if 'Date' not in response.headers:
|
||||
response.headers['Date'] = formatdate(usegmt=1)
|
||||
response.headers['Date'] = formatdate(usegmt=True)
|
||||
|
||||
# Do not validate first-hand responses
|
||||
cachedresponse = request.meta.pop('cached_response', None)
|
||||
|
|
@ -102,13 +114,18 @@ class HttpCacheMiddleware:
|
|||
self._cache_response(spider, response, request, cachedresponse)
|
||||
return response
|
||||
|
||||
def process_exception(self, request, exception, spider):
|
||||
def process_exception(
|
||||
self, request: Request, exception: Exception, spider: Spider
|
||||
) -> Optional[Response]:
|
||||
cachedresponse = request.meta.pop('cached_response', None)
|
||||
if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS):
|
||||
self.stats.inc_value('httpcache/errorrecovery', spider=spider)
|
||||
return cachedresponse
|
||||
return None
|
||||
|
||||
def _cache_response(self, spider, response, request, cachedresponse):
|
||||
def _cache_response(
|
||||
self, spider: Spider, response: Response, request: Request, cachedresponse: Optional[Response]
|
||||
) -> None:
|
||||
if self.policy.should_cache_response(response, request):
|
||||
self.stats.inc_value('httpcache/store', spider=spider)
|
||||
self.storage.store_response(spider, request, response)
|
||||
|
|
|
|||
|
|
@ -24,17 +24,34 @@ from scrapy.utils.conf import feed_complete_default_values_from_settings
|
|||
from scrapy.utils.ftp import ftp_store_file
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
from scrapy.utils.misc import create_instance, load_object
|
||||
from scrapy.utils.python import without_none_values
|
||||
from scrapy.utils.python import get_func_args, without_none_values
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
|
||||
argument_names = get_func_args(builder)
|
||||
if 'feed_options' in argument_names:
|
||||
kwargs['feed_options'] = feed_options
|
||||
else:
|
||||
warnings.warn(
|
||||
"{} does not support the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove this "
|
||||
"warning. This parameter will become mandatory in a future "
|
||||
"version of Scrapy."
|
||||
.format(builder.__qualname__),
|
||||
category=ScrapyDeprecationWarning
|
||||
)
|
||||
return builder(*preargs, uri, *args, **kwargs)
|
||||
|
||||
|
||||
class IFeedStorage(Interface):
|
||||
"""Interface that all Feed Storages must implement"""
|
||||
|
||||
def __init__(uri):
|
||||
"""Initialize the storage with the parameters given in the URI"""
|
||||
def __init__(uri, *, feed_options=None):
|
||||
"""Initialize the storage with the parameters given in the URI and the
|
||||
feed-specific options (see :setting:`FEEDS`)"""
|
||||
|
||||
def open(spider):
|
||||
"""Open the storage for the given spider. It must return a file-like
|
||||
|
|
@ -64,10 +81,15 @@ class BlockingFeedStorage:
|
|||
@implementer(IFeedStorage)
|
||||
class StdoutFeedStorage:
|
||||
|
||||
def __init__(self, uri, _stdout=None):
|
||||
def __init__(self, uri, _stdout=None, *, feed_options=None):
|
||||
if not _stdout:
|
||||
_stdout = sys.stdout.buffer
|
||||
self._stdout = _stdout
|
||||
if feed_options and feed_options.get('overwrite', False) is True:
|
||||
logger.warning('Standard output (stdout) storage does not support '
|
||||
'overwriting. To suppress this warning, remove the '
|
||||
'overwrite option from your FEEDS setting, or set '
|
||||
'it to False.')
|
||||
|
||||
def open(self, spider):
|
||||
return self._stdout
|
||||
|
|
@ -79,14 +101,16 @@ class StdoutFeedStorage:
|
|||
@implementer(IFeedStorage)
|
||||
class FileFeedStorage:
|
||||
|
||||
def __init__(self, uri):
|
||||
def __init__(self, uri, *, feed_options=None):
|
||||
self.path = file_uri_to_path(uri)
|
||||
feed_options = feed_options or {}
|
||||
self.write_mode = 'wb' if feed_options.get('overwrite', False) else 'ab'
|
||||
|
||||
def open(self, spider):
|
||||
dirname = os.path.dirname(self.path)
|
||||
if dirname and not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
return open(self.path, 'ab')
|
||||
return open(self.path, self.write_mode)
|
||||
|
||||
def store(self, file):
|
||||
file.close()
|
||||
|
|
@ -94,24 +118,8 @@ class FileFeedStorage:
|
|||
|
||||
class S3FeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri, access_key=None, secret_key=None, acl=None):
|
||||
# BEGIN Backward compatibility for initialising without keys (and
|
||||
# without using from_crawler)
|
||||
no_defaults = access_key is None and secret_key is None
|
||||
if no_defaults:
|
||||
from scrapy.utils.project import get_project_settings
|
||||
settings = get_project_settings()
|
||||
if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings:
|
||||
warnings.warn(
|
||||
"Initialising `scrapy.extensions.feedexport.S3FeedStorage` "
|
||||
"without AWS keys is deprecated. Please supply credentials or "
|
||||
"use the `from_crawler()` constructor.",
|
||||
category=ScrapyDeprecationWarning,
|
||||
stacklevel=2
|
||||
)
|
||||
access_key = settings['AWS_ACCESS_KEY_ID']
|
||||
secret_key = settings['AWS_SECRET_ACCESS_KEY']
|
||||
# END Backward compatibility
|
||||
def __init__(self, uri, access_key=None, secret_key=None, acl=None, *,
|
||||
feed_options=None):
|
||||
u = urlparse(uri)
|
||||
self.bucketname = u.hostname
|
||||
self.access_key = u.username or access_key
|
||||
|
|
@ -128,14 +136,20 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
else:
|
||||
import boto
|
||||
self.connect_s3 = boto.connect_s3
|
||||
if feed_options and feed_options.get('overwrite', True) is False:
|
||||
logger.warning('S3 does not support appending to files. To '
|
||||
'suppress this warning, remove the overwrite '
|
||||
'option from your FEEDS setting or set it to True.')
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return cls(
|
||||
uri=uri,
|
||||
def from_crawler(cls, crawler, uri, *, feed_options=None):
|
||||
return build_storage(
|
||||
cls,
|
||||
uri,
|
||||
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
|
||||
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
|
||||
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None
|
||||
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
|
||||
feed_options=feed_options,
|
||||
)
|
||||
|
||||
def _store_in_thread(self, file):
|
||||
|
|
@ -152,6 +166,7 @@ class S3FeedStorage(BlockingFeedStorage):
|
|||
kwargs = {'policy': self.acl} if self.acl else {}
|
||||
key.set_contents_from_file(file, **kwargs)
|
||||
key.close()
|
||||
file.close()
|
||||
|
||||
|
||||
class GCSFeedStorage(BlockingFeedStorage):
|
||||
|
|
@ -182,27 +197,31 @@ class GCSFeedStorage(BlockingFeedStorage):
|
|||
|
||||
class FTPFeedStorage(BlockingFeedStorage):
|
||||
|
||||
def __init__(self, uri, use_active_mode=False):
|
||||
def __init__(self, uri, use_active_mode=False, *, feed_options=None):
|
||||
u = urlparse(uri)
|
||||
self.host = u.hostname
|
||||
self.port = int(u.port or '21')
|
||||
self.username = u.username
|
||||
self.password = unquote(u.password)
|
||||
self.password = unquote(u.password or '')
|
||||
self.path = u.path
|
||||
self.use_active_mode = use_active_mode
|
||||
self.overwrite = not feed_options or feed_options.get('overwrite', True)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return cls(
|
||||
uri=uri,
|
||||
use_active_mode=crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE')
|
||||
def from_crawler(cls, crawler, uri, *, feed_options=None):
|
||||
return build_storage(
|
||||
cls,
|
||||
uri,
|
||||
crawler.settings.getbool('FEED_STORAGE_FTP_ACTIVE'),
|
||||
feed_options=feed_options,
|
||||
)
|
||||
|
||||
def _store_in_thread(self, file):
|
||||
ftp_store_file(
|
||||
path=self.path, file=file, host=self.host,
|
||||
port=self.port, username=self.username,
|
||||
password=self.password, use_active_mode=self.use_active_mode
|
||||
password=self.password, use_active_mode=self.use_active_mode,
|
||||
overwrite=self.overwrite,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -259,32 +278,32 @@ class FeedExporter:
|
|||
category=ScrapyDeprecationWarning, stacklevel=2,
|
||||
)
|
||||
uri = str(self.settings['FEED_URI']) # handle pathlib.Path objects
|
||||
feed = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
|
||||
feed_options = {'format': self.settings.get('FEED_FORMAT', 'jsonlines')}
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
|
||||
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
|
||||
|
||||
# 'FEEDS' setting takes precedence over 'FEED_URI'
|
||||
for uri, feed in self.settings.getdict('FEEDS').items():
|
||||
for uri, feed_options in self.settings.getdict('FEEDS').items():
|
||||
uri = str(uri) # handle pathlib.Path objects
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed, self.settings)
|
||||
self.feeds[uri] = feed_complete_default_values_from_settings(feed_options, self.settings)
|
||||
|
||||
self.storages = self._load_components('FEED_STORAGES')
|
||||
self.exporters = self._load_components('FEED_EXPORTERS')
|
||||
for uri, feed in self.feeds.items():
|
||||
if not self._storage_supported(uri):
|
||||
for uri, feed_options in self.feeds.items():
|
||||
if not self._storage_supported(uri, feed_options):
|
||||
raise NotConfigured
|
||||
if not self._settings_are_valid():
|
||||
raise NotConfigured
|
||||
if not self._exporter_supported(feed['format']):
|
||||
if not self._exporter_supported(feed_options['format']):
|
||||
raise NotConfigured
|
||||
|
||||
def open_spider(self, spider):
|
||||
for uri, feed in self.feeds.items():
|
||||
uri_params = self._get_uri_params(spider, feed['uri_params'])
|
||||
for uri, feed_options in self.feeds.items():
|
||||
uri_params = self._get_uri_params(spider, feed_options['uri_params'])
|
||||
self.slots.append(self._start_new_batch(
|
||||
batch_id=1,
|
||||
uri=uri % uri_params,
|
||||
feed=feed,
|
||||
feed_options=feed_options,
|
||||
spider=spider,
|
||||
uri_template=uri,
|
||||
))
|
||||
|
|
@ -323,32 +342,32 @@ class FeedExporter:
|
|||
)
|
||||
return d
|
||||
|
||||
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
|
||||
def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
|
||||
"""
|
||||
Redirect the output data stream to a new file.
|
||||
Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
|
||||
:param batch_id: sequence number of current batch
|
||||
:param uri: uri of the new batch to start
|
||||
:param feed: dict with parameters of feed
|
||||
:param feed_options: dict with parameters of feed
|
||||
:param spider: user spider
|
||||
:param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri
|
||||
"""
|
||||
storage = self._get_storage(uri)
|
||||
storage = self._get_storage(uri, feed_options)
|
||||
file = storage.open(spider)
|
||||
exporter = self._get_exporter(
|
||||
file=file,
|
||||
format=feed['format'],
|
||||
fields_to_export=feed['fields'],
|
||||
encoding=feed['encoding'],
|
||||
indent=feed['indent'],
|
||||
format=feed_options['format'],
|
||||
fields_to_export=feed_options['fields'],
|
||||
encoding=feed_options['encoding'],
|
||||
indent=feed_options['indent'],
|
||||
)
|
||||
slot = _FeedSlot(
|
||||
file=file,
|
||||
exporter=exporter,
|
||||
storage=storage,
|
||||
uri=uri,
|
||||
format=feed['format'],
|
||||
store_empty=feed['store_empty'],
|
||||
format=feed_options['format'],
|
||||
store_empty=feed_options['store_empty'],
|
||||
batch_id=batch_id,
|
||||
uri_template=uri_template,
|
||||
)
|
||||
|
|
@ -372,7 +391,7 @@ class FeedExporter:
|
|||
slots.append(self._start_new_batch(
|
||||
batch_id=slot.batch_id + 1,
|
||||
uri=slot.uri_template % uri_params,
|
||||
feed=self.feeds[slot.uri_template],
|
||||
feed_options=self.feeds[slot.uri_template],
|
||||
spider=spider,
|
||||
uri_template=slot.uri_template,
|
||||
))
|
||||
|
|
@ -411,11 +430,11 @@ class FeedExporter:
|
|||
return False
|
||||
return True
|
||||
|
||||
def _storage_supported(self, uri):
|
||||
def _storage_supported(self, uri, feed_options):
|
||||
scheme = urlparse(uri).scheme
|
||||
if scheme in self.storages:
|
||||
try:
|
||||
self._get_storage(uri)
|
||||
self._get_storage(uri, feed_options)
|
||||
return True
|
||||
except NotConfigured as e:
|
||||
logger.error("Disabled feed storage scheme: %(scheme)s. "
|
||||
|
|
@ -433,8 +452,30 @@ class FeedExporter:
|
|||
def _get_exporter(self, file, format, *args, **kwargs):
|
||||
return self._get_instance(self.exporters[format], file, *args, **kwargs)
|
||||
|
||||
def _get_storage(self, uri):
|
||||
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
|
||||
def _get_storage(self, uri, feed_options):
|
||||
"""Fork of create_instance specific to feed storage classes
|
||||
|
||||
It supports not passing the *feed_options* parameters to classes that
|
||||
do not support it, and issuing a deprecation warning instead.
|
||||
"""
|
||||
feedcls = self.storages[urlparse(uri).scheme]
|
||||
crawler = getattr(self, 'crawler', None)
|
||||
|
||||
def build_instance(builder, *preargs):
|
||||
return build_storage(builder, uri, preargs=preargs)
|
||||
|
||||
if crawler and hasattr(feedcls, 'from_crawler'):
|
||||
instance = build_instance(feedcls.from_crawler, crawler)
|
||||
method_name = 'from_crawler'
|
||||
elif hasattr(feedcls, 'from_settings'):
|
||||
instance = build_instance(feedcls.from_settings, self.settings)
|
||||
method_name = 'from_settings'
|
||||
else:
|
||||
instance = build_instance(feedcls)
|
||||
method_name = '__new__'
|
||||
if instance is None:
|
||||
raise TypeError("%s.%s returned None" % (feedcls.__qualname__, method_name))
|
||||
return instance
|
||||
|
||||
def _get_uri_params(self, spider, uri_params, slot=None):
|
||||
params = {}
|
||||
|
|
|
|||
|
|
@ -409,7 +409,7 @@ class FilesPipeline(MediaPipeline):
|
|||
store_cls = self.STORE_SCHEMES[scheme]
|
||||
return store_cls(uri)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
def _onsuccess(result):
|
||||
if not result:
|
||||
return # returning None force download
|
||||
|
|
@ -436,7 +436,7 @@ class FilesPipeline(MediaPipeline):
|
|||
checksum = result.get('checksum', None)
|
||||
return {'url': request.url, 'path': path, 'checksum': checksum, 'status': 'uptodate'}
|
||||
|
||||
path = self.file_path(request, info=info)
|
||||
path = self.file_path(request, info=info, item=item)
|
||||
dfd = defer.maybeDeferred(self.store.stat_file, path, info)
|
||||
dfd.addCallbacks(_onsuccess, lambda _: None)
|
||||
dfd.addErrback(
|
||||
|
|
@ -460,7 +460,7 @@ class FilesPipeline(MediaPipeline):
|
|||
|
||||
raise FileException
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
referer = referer_str(request)
|
||||
|
||||
if response.status != 200:
|
||||
|
|
@ -492,8 +492,8 @@ class FilesPipeline(MediaPipeline):
|
|||
self.inc_stats(info.spider, status)
|
||||
|
||||
try:
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
checksum = self.file_downloaded(response, request, info)
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
checksum = self.file_downloaded(response, request, info, item=item)
|
||||
except FileException as exc:
|
||||
logger.warning(
|
||||
'File (error): Error processing file from %(request)s '
|
||||
|
|
@ -522,8 +522,8 @@ class FilesPipeline(MediaPipeline):
|
|||
urls = ItemAdapter(item).get(self.files_urls_field, [])
|
||||
return [Request(u) for u in urls]
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
def file_downloaded(self, response, request, info, *, item=None):
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
buf = BytesIO(response.body)
|
||||
checksum = md5sum(buf)
|
||||
buf.seek(0)
|
||||
|
|
@ -535,7 +535,7 @@ class FilesPipeline(MediaPipeline):
|
|||
ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
media_ext = os.path.splitext(request.url)[1]
|
||||
# Handles empty and wild extensions by trying to guess the
|
||||
|
|
|
|||
|
|
@ -103,12 +103,12 @@ class ImagesPipeline(FilesPipeline):
|
|||
store_uri = settings['IMAGES_STORE']
|
||||
return cls(store_uri, settings=settings)
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
return self.image_downloaded(response, request, info)
|
||||
def file_downloaded(self, response, request, info, *, item=None):
|
||||
return self.image_downloaded(response, request, info, item=item)
|
||||
|
||||
def image_downloaded(self, response, request, info):
|
||||
def image_downloaded(self, response, request, info, *, item=None):
|
||||
checksum = None
|
||||
for path, image, buf in self.get_images(response, request, info):
|
||||
for path, image, buf in self.get_images(response, request, info, item=item):
|
||||
if checksum is None:
|
||||
buf.seek(0)
|
||||
checksum = md5sum(buf)
|
||||
|
|
@ -119,8 +119,8 @@ class ImagesPipeline(FilesPipeline):
|
|||
headers={'Content-Type': 'image/jpeg'})
|
||||
return checksum
|
||||
|
||||
def get_images(self, response, request, info):
|
||||
path = self.file_path(request, response=response, info=info)
|
||||
def get_images(self, response, request, info, *, item=None):
|
||||
path = self.file_path(request, response=response, info=info, item=item)
|
||||
orig_image = Image.open(BytesIO(response.body))
|
||||
|
||||
width, height = orig_image.size
|
||||
|
|
@ -166,7 +166,7 @@ class ImagesPipeline(FilesPipeline):
|
|||
ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
|
||||
return 'full/%s.jpg' % (image_guid)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,12 +1,16 @@
|
|||
import functools
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from inspect import signature
|
||||
from warnings import warn
|
||||
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.utils.datatypes import SequenceExclude
|
||||
from scrapy.utils.defer import mustbe_deferred, defer_result
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
|
@ -27,6 +31,7 @@ class MediaPipeline:
|
|||
|
||||
def __init__(self, download_func=None, settings=None):
|
||||
self.download_func = download_func
|
||||
self._expects_item = {}
|
||||
|
||||
if isinstance(settings, dict) or settings is None:
|
||||
settings = Settings(settings)
|
||||
|
|
@ -38,6 +43,9 @@ class MediaPipeline:
|
|||
)
|
||||
self._handle_statuses(self.allow_redirects)
|
||||
|
||||
# Check if deprecated methods are being used and make them compatible
|
||||
self._make_compatible()
|
||||
|
||||
def _handle_statuses(self, allow_redirects):
|
||||
self.handle_httpstatus_list = None
|
||||
if allow_redirects:
|
||||
|
|
@ -77,11 +85,11 @@ class MediaPipeline:
|
|||
def process_item(self, item, spider):
|
||||
info = self.spiderinfo
|
||||
requests = arg_to_iter(self.get_media_requests(item, info))
|
||||
dlist = [self._process_request(r, info) for r in requests]
|
||||
dlist = [self._process_request(r, info, item) for r in requests]
|
||||
dfd = DeferredList(dlist, consumeErrors=1)
|
||||
return dfd.addCallback(self.item_completed, item, info)
|
||||
|
||||
def _process_request(self, request, info):
|
||||
def _process_request(self, request, info, item):
|
||||
fp = request_fingerprint(request)
|
||||
cb = request.callback or (lambda _: _)
|
||||
eb = request.errback
|
||||
|
|
@ -102,34 +110,73 @@ class MediaPipeline:
|
|||
|
||||
# Download request checking media_to_download hook output first
|
||||
info.downloading.add(fp)
|
||||
dfd = mustbe_deferred(self.media_to_download, request, info)
|
||||
dfd.addCallback(self._check_media_to_download, request, info)
|
||||
dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
|
||||
dfd.addCallback(self._check_media_to_download, request, info, item=item)
|
||||
dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
|
||||
dfd.addErrback(lambda f: logger.error(
|
||||
f.value, exc_info=failure_to_exc_info(f), extra={'spider': info.spider})
|
||||
)
|
||||
return dfd.addBoth(lambda _: wad) # it must return wad at last
|
||||
|
||||
def _make_compatible(self):
|
||||
"""Make overridable methods of MediaPipeline and subclasses backwards compatible"""
|
||||
methods = [
|
||||
"file_path", "media_to_download", "media_downloaded",
|
||||
"file_downloaded", "image_downloaded", "get_images"
|
||||
]
|
||||
|
||||
for method_name in methods:
|
||||
method = getattr(self, method_name, None)
|
||||
if callable(method):
|
||||
setattr(self, method_name, self._compatible(method))
|
||||
|
||||
def _compatible(self, func):
|
||||
"""Wrapper for overridable methods to allow backwards compatibility"""
|
||||
self._check_signature(func)
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
if self._expects_item[func.__name__]:
|
||||
return func(*args, **kwargs)
|
||||
|
||||
kwargs.pop('item', None)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
def _check_signature(self, func):
|
||||
sig = signature(func)
|
||||
self._expects_item[func.__name__] = True
|
||||
|
||||
if 'item' not in sig.parameters:
|
||||
old_params = str(sig)[1:-1]
|
||||
new_params = old_params + ", *, item=None"
|
||||
warn('%s(self, %s) is deprecated, '
|
||||
'please use %s(self, %s)'
|
||||
% (func.__name__, old_params, func.__name__, new_params),
|
||||
ScrapyDeprecationWarning, stacklevel=2)
|
||||
self._expects_item[func.__name__] = False
|
||||
|
||||
def _modify_media_request(self, request):
|
||||
if self.handle_httpstatus_list:
|
||||
request.meta['handle_httpstatus_list'] = self.handle_httpstatus_list
|
||||
else:
|
||||
request.meta['handle_httpstatus_all'] = True
|
||||
|
||||
def _check_media_to_download(self, result, request, info):
|
||||
def _check_media_to_download(self, result, request, info, item):
|
||||
if result is not None:
|
||||
return result
|
||||
if self.download_func:
|
||||
# this ugly code was left only to support tests. TODO: remove
|
||||
dfd = mustbe_deferred(self.download_func, request, info.spider)
|
||||
dfd.addCallbacks(
|
||||
callback=self.media_downloaded, callbackArgs=(request, info),
|
||||
callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
|
||||
errback=self.media_failed, errbackArgs=(request, info))
|
||||
else:
|
||||
self._modify_media_request(request)
|
||||
dfd = self.crawler.engine.download(request, info.spider)
|
||||
dfd.addCallbacks(
|
||||
callback=self.media_downloaded, callbackArgs=(request, info),
|
||||
callback=self.media_downloaded, callbackArgs=(request, info), callbackKeywords={'item': item},
|
||||
errback=self.media_failed, errbackArgs=(request, info))
|
||||
return dfd
|
||||
|
||||
|
|
@ -171,7 +218,7 @@ class MediaPipeline:
|
|||
defer_result(result).chainDeferred(wad)
|
||||
|
||||
# Overridable Interface
|
||||
def media_to_download(self, request, info):
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
"""Check request before starting download"""
|
||||
pass
|
||||
|
||||
|
|
@ -179,7 +226,7 @@ class MediaPipeline:
|
|||
"""Returns the media requests to download"""
|
||||
pass
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
"""Handler for success downloads"""
|
||||
return response
|
||||
|
||||
|
|
@ -199,3 +246,7 @@ class MediaPipeline:
|
|||
extra={'spider': info.spider}
|
||||
)
|
||||
return item
|
||||
|
||||
def file_path(self, request, response=None, info=None, *, item=None):
|
||||
"""Returns the path where downloaded media should be stored"""
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -47,10 +47,10 @@ class RobotParser(metaclass=ABCMeta):
|
|||
"""Return ``True`` if ``user_agent`` is allowed to crawl ``url``, otherwise return ``False``.
|
||||
|
||||
:param url: Absolute URL
|
||||
:type url: string
|
||||
:type url: str
|
||||
|
||||
:param user_agent: User agent
|
||||
:type user_agent: string
|
||||
:type user_agent: str
|
||||
"""
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -98,10 +98,10 @@ class BaseSettings(MutableMapping):
|
|||
Get a setting value without affecting its original type.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
return self[name] if self[name] is not None else default
|
||||
|
||||
|
|
@ -116,10 +116,10 @@ class BaseSettings(MutableMapping):
|
|||
``'0'`` will return ``False`` when using this method.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
got = self.get(name, default)
|
||||
try:
|
||||
|
|
@ -138,10 +138,10 @@ class BaseSettings(MutableMapping):
|
|||
Get a setting value as an int.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
return int(self.get(name, default))
|
||||
|
||||
|
|
@ -150,10 +150,10 @@ class BaseSettings(MutableMapping):
|
|||
Get a setting value as a float.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
return float(self.get(name, default))
|
||||
|
||||
|
|
@ -166,10 +166,10 @@ class BaseSettings(MutableMapping):
|
|||
``'one,two'`` will return a list ['one', 'two'] when using this method.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
value = self.get(name, default or [])
|
||||
if isinstance(value, str):
|
||||
|
|
@ -187,10 +187,10 @@ class BaseSettings(MutableMapping):
|
|||
and losing all information about priority and mutability.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param default: the value to return if no setting is found
|
||||
:type default: any
|
||||
:type default: object
|
||||
"""
|
||||
value = self.get(name, default or {})
|
||||
if isinstance(value, str):
|
||||
|
|
@ -202,7 +202,7 @@ class BaseSettings(MutableMapping):
|
|||
counterpart.
|
||||
|
||||
:param name: name of the dictionary-like setting
|
||||
:type name: string
|
||||
:type name: str
|
||||
"""
|
||||
compbs = BaseSettings()
|
||||
compbs.update(self[name + '_BASE'])
|
||||
|
|
@ -215,7 +215,7 @@ class BaseSettings(MutableMapping):
|
|||
the given ``name`` does not exist.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
"""
|
||||
if name not in self:
|
||||
return None
|
||||
|
|
@ -245,14 +245,14 @@ class BaseSettings(MutableMapping):
|
|||
otherwise they won't have any effect.
|
||||
|
||||
:param name: the setting name
|
||||
:type name: string
|
||||
:type name: str
|
||||
|
||||
:param value: the value to associate with the setting
|
||||
:type value: any
|
||||
:type value: object
|
||||
|
||||
:param priority: the priority of the setting. Should be a key of
|
||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||
:type priority: string or int
|
||||
:type priority: str or int
|
||||
"""
|
||||
self._assert_mutability()
|
||||
priority = get_settings_priority(priority)
|
||||
|
|
@ -276,11 +276,11 @@ class BaseSettings(MutableMapping):
|
|||
uppercase variable of ``module`` with the provided ``priority``.
|
||||
|
||||
:param module: the module or the path of the module
|
||||
:type module: module object or string
|
||||
:type module: types.ModuleType or str
|
||||
|
||||
:param priority: the priority of the settings. Should be a key of
|
||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||
:type priority: string or int
|
||||
:type priority: str or int
|
||||
"""
|
||||
self._assert_mutability()
|
||||
if isinstance(module, str):
|
||||
|
|
@ -309,7 +309,7 @@ class BaseSettings(MutableMapping):
|
|||
|
||||
:param priority: the priority of the settings. Should be a key of
|
||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||
:type priority: string or int
|
||||
:type priority: str or int
|
||||
"""
|
||||
self._assert_mutability()
|
||||
if isinstance(values, str):
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ from os.path import join, abspath, dirname
|
|||
|
||||
AJAXCRAWL_ENABLED = False
|
||||
|
||||
ASYNCIO_EVENT_LOOP = None
|
||||
|
||||
AUTOTHROTTLE_ENABLED = False
|
||||
AUTOTHROTTLE_DEBUG = False
|
||||
AUTOTHROTTLE_MAX_DELAY = 60.0
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class SignalManager:
|
|||
section.
|
||||
|
||||
:param receiver: the function to be connected
|
||||
:type receiver: callable
|
||||
:type receiver: collections.abc.Callable
|
||||
|
||||
:param signal: the signal to connect to
|
||||
:type signal: object
|
||||
|
|
|
|||
|
|
@ -127,7 +127,8 @@ def feed_complete_default_values_from_settings(feed, settings):
|
|||
return out
|
||||
|
||||
|
||||
def feed_process_params_from_cli(settings, output, output_format=None):
|
||||
def feed_process_params_from_cli(settings, output, output_format=None,
|
||||
overwrite_output=None):
|
||||
"""
|
||||
Receives feed export params (from the 'crawl' or 'runspider' commands),
|
||||
checks for inconsistencies in their quantities and returns a dictionary
|
||||
|
|
@ -139,22 +140,39 @@ def feed_process_params_from_cli(settings, output, output_format=None):
|
|||
|
||||
def check_valid_format(output_format):
|
||||
if output_format not in valid_output_formats:
|
||||
raise UsageError("Unrecognized output format '%s', set one after a"
|
||||
" colon using the -o option (i.e. -o <URI>:<FORMAT>)"
|
||||
" or as a file extension, from the supported list %s" %
|
||||
(output_format, tuple(valid_output_formats)))
|
||||
raise UsageError(
|
||||
"Unrecognized output format '%s'. Set a supported one (%s) "
|
||||
"after a colon at the end of the output URI (i.e. -o/-O "
|
||||
"<URI>:<FORMAT>) or as a file extension." % (
|
||||
output_format,
|
||||
tuple(valid_output_formats),
|
||||
)
|
||||
)
|
||||
|
||||
overwrite = False
|
||||
if overwrite_output:
|
||||
if output:
|
||||
raise UsageError(
|
||||
"Please use only one of -o/--output and -O/--overwrite-output"
|
||||
)
|
||||
output = overwrite_output
|
||||
overwrite = True
|
||||
|
||||
if output_format:
|
||||
if len(output) == 1:
|
||||
check_valid_format(output_format)
|
||||
warnings.warn('The -t command line option is deprecated in favor'
|
||||
' of specifying the output format within the -o'
|
||||
' option, please check the -o option docs for more details',
|
||||
category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
message = (
|
||||
'The -t command line option is deprecated in favor of '
|
||||
'specifying the output format within the output URI. See the '
|
||||
'documentation of the -o and -O options for more information.',
|
||||
)
|
||||
warnings.warn(message, ScrapyDeprecationWarning, stacklevel=2)
|
||||
return {output[0]: {'format': output_format}}
|
||||
else:
|
||||
raise UsageError('The -t command line option cannot be used if multiple'
|
||||
' output files are specified with the -o option')
|
||||
raise UsageError(
|
||||
'The -t command-line option cannot be used if multiple output '
|
||||
'URIs are specified'
|
||||
)
|
||||
|
||||
result = {}
|
||||
for element in output:
|
||||
|
|
@ -168,8 +186,10 @@ def feed_process_params_from_cli(settings, output, output_format=None):
|
|||
feed_uri = 'stdout:'
|
||||
check_valid_format(feed_format)
|
||||
result[feed_uri] = {'format': feed_format}
|
||||
if overwrite:
|
||||
result[feed_uri]['overwrite'] = True
|
||||
|
||||
# FEEDS setting should take precedence over the -o and -t CLI options
|
||||
# FEEDS setting should take precedence over the matching CLI options
|
||||
result.update(settings.getdict('FEEDS'))
|
||||
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ DEPRECATION_RULES = [
|
|||
def update_classpath(path):
|
||||
"""Update a deprecated path from an object with its new location"""
|
||||
for prefix, replacement in DEPRECATION_RULES:
|
||||
if path.startswith(prefix):
|
||||
if isinstance(path, str) and path.startswith(prefix):
|
||||
new_path = path.replace(prefix, replacement, 1)
|
||||
warnings.warn("`{}` class is deprecated, use `{}` instead".format(path, new_path),
|
||||
ScrapyDeprecationWarning)
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ def ftp_makedirs_cwd(ftp, path, first_call=True):
|
|||
|
||||
def ftp_store_file(
|
||||
*, path, file, host, port,
|
||||
username, password, use_active_mode=False):
|
||||
username, password, use_active_mode=False, overwrite=True):
|
||||
"""Opens a FTP connection with passed credentials,sets current directory
|
||||
to the directory extracted from given path, then uploads the file to server
|
||||
"""
|
||||
|
|
@ -32,4 +32,6 @@ def ftp_store_file(
|
|||
file.seek(0)
|
||||
dirname, filename = posixpath.split(path)
|
||||
ftp_makedirs_cwd(ftp, dirname)
|
||||
ftp.storbinary('STOR %s' % filename, file)
|
||||
command = 'STOR' if overwrite else 'APPE'
|
||||
ftp.storbinary('%s %s' % (command, filename), file)
|
||||
file.close()
|
||||
|
|
|
|||
|
|
@ -150,6 +150,13 @@ def log_scrapy_info(settings):
|
|||
logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
|
||||
from twisted.internet import reactor
|
||||
logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
|
||||
from twisted.internet import asyncioreactor
|
||||
if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
|
||||
logger.debug(
|
||||
"Using asyncio event loop: %s.%s",
|
||||
reactor._asyncioEventloop.__module__,
|
||||
reactor._asyncioEventloop.__class__.__name__,
|
||||
)
|
||||
|
||||
|
||||
class StreamLogger:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import os
|
|||
import re
|
||||
import hashlib
|
||||
import warnings
|
||||
from collections import deque
|
||||
from contextlib import contextmanager
|
||||
from importlib import import_module
|
||||
from pkgutil import iter_modules
|
||||
|
|
@ -38,10 +39,20 @@ def arg_to_iter(arg):
|
|||
def load_object(path):
|
||||
"""Load an object given its absolute object path, and return it.
|
||||
|
||||
object can be the import path of a class, function, variable or an
|
||||
instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
|
||||
The object can be the import path of a class, function, variable or an
|
||||
instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.
|
||||
|
||||
If ``path`` is not a string, but is a callable object, such as a class or
|
||||
a function, then return it as is.
|
||||
"""
|
||||
|
||||
if not isinstance(path, str):
|
||||
if callable(path):
|
||||
return path
|
||||
else:
|
||||
raise TypeError("Unexpected argument type, expected string "
|
||||
"or object, got: %s" % type(path))
|
||||
|
||||
try:
|
||||
dot = path.rindex('.')
|
||||
except ValueError:
|
||||
|
|
@ -184,6 +195,22 @@ def set_environ(**kwargs):
|
|||
os.environ[k] = v
|
||||
|
||||
|
||||
def walk_callable(node):
|
||||
"""Similar to ``ast.walk``, but walks only function body and skips nested
|
||||
functions defined within the node.
|
||||
"""
|
||||
todo = deque([node])
|
||||
walked_func_def = False
|
||||
while todo:
|
||||
node = todo.popleft()
|
||||
if isinstance(node, ast.FunctionDef):
|
||||
if walked_func_def:
|
||||
continue
|
||||
walked_func_def = True
|
||||
todo.extend(ast.iter_child_nodes(node))
|
||||
yield node
|
||||
|
||||
|
||||
_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
|
||||
|
||||
|
||||
|
|
@ -201,7 +228,7 @@ def is_generator_with_return_value(callable):
|
|||
|
||||
if inspect.isgeneratorfunction(callable):
|
||||
tree = ast.parse(dedent(inspect.getsource(callable)))
|
||||
for node in ast.walk(tree):
|
||||
for node in walk_callable(tree):
|
||||
if isinstance(node, ast.Return) and not returns_none(node):
|
||||
_generator_callbacks_cache[callable] = True
|
||||
return _generator_callbacks_cache[callable]
|
||||
|
|
|
|||
|
|
@ -198,7 +198,8 @@ def _getargspec_py23(func):
|
|||
def get_func_args(func, stripself=False):
|
||||
"""Return the argument name list of a callable"""
|
||||
if inspect.isfunction(func):
|
||||
func_args, _, _, _ = _getargspec_py23(func)
|
||||
spec = inspect.getfullargspec(func)
|
||||
func_args = spec.args + spec.kwonlyargs
|
||||
elif inspect.isclass(func):
|
||||
return get_func_args(func.__init__, True)
|
||||
elif inspect.ismethod(func):
|
||||
|
|
|
|||
|
|
@ -50,13 +50,19 @@ class CallLaterOnce:
|
|||
return self._func(*self._a, **self._kw)
|
||||
|
||||
|
||||
def install_reactor(reactor_path):
|
||||
def install_reactor(reactor_path, event_loop_path=None):
|
||||
"""Installs the :mod:`~twisted.internet.reactor` with the specified
|
||||
import path."""
|
||||
import path. Also installs the asyncio event loop with the specified import
|
||||
path if the asyncio reactor is enabled"""
|
||||
reactor_class = load_object(reactor_path)
|
||||
if reactor_class is asyncioreactor.AsyncioSelectorReactor:
|
||||
with suppress(error.ReactorAlreadyInstalledError):
|
||||
asyncioreactor.install(asyncio.get_event_loop())
|
||||
if event_loop_path is not None:
|
||||
event_loop_class = load_object(event_loop_path)
|
||||
event_loop = event_loop_class()
|
||||
else:
|
||||
event_loop = asyncio.new_event_loop()
|
||||
asyncioreactor.install(eventloop=event_loop)
|
||||
else:
|
||||
*module, _ = reactor_path.split(".")
|
||||
installer_path = module + ["install"]
|
||||
|
|
|
|||
|
|
@ -71,25 +71,20 @@ def request_from_dict(d, spider=None):
|
|||
|
||||
|
||||
def _find_method(obj, func):
|
||||
if obj:
|
||||
try:
|
||||
func_self = func.__self__
|
||||
except AttributeError: # func has no __self__
|
||||
pass
|
||||
else:
|
||||
if func_self is obj:
|
||||
members = inspect.getmembers(obj, predicate=inspect.ismethod)
|
||||
for name, obj_func in members:
|
||||
# We need to use __func__ to access the original
|
||||
# function object because instance method objects
|
||||
# are generated each time attribute is retrieved from
|
||||
# instance.
|
||||
#
|
||||
# Reference: The standard type hierarchy
|
||||
# https://docs.python.org/3/reference/datamodel.html
|
||||
if obj_func.__func__ is func.__func__:
|
||||
return name
|
||||
raise ValueError("Function %s is not a method of: %s" % (func, obj))
|
||||
# Only instance methods contain ``__func__``
|
||||
if obj and hasattr(func, '__func__'):
|
||||
members = inspect.getmembers(obj, predicate=inspect.ismethod)
|
||||
for name, obj_func in members:
|
||||
# We need to use __func__ to access the original
|
||||
# function object because instance method objects
|
||||
# are generated each time attribute is retrieved from
|
||||
# instance.
|
||||
#
|
||||
# Reference: The standard type hierarchy
|
||||
# https://docs.python.org/3/reference/datamodel.html
|
||||
if obj_func.__func__ is func.__func__:
|
||||
return name
|
||||
raise ValueError("Function %s is not an instance method in: %s" % (func, obj))
|
||||
|
||||
|
||||
def _get_method(obj, name):
|
||||
|
|
|
|||
|
|
@ -130,6 +130,9 @@ ignore_errors = True
|
|||
[mypy-tests.test_pipelines]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_attribute_binding]
|
||||
ignore_errors = True
|
||||
|
||||
[mypy-tests.test_request_cb_kwargs]
|
||||
ignore_errors = True
|
||||
|
||||
|
|
|
|||
2
setup.py
2
setup.py
|
|
@ -41,7 +41,7 @@ if has_environment_marker_platform_impl_support():
|
|||
]
|
||||
extras_require[':platform_python_implementation == "PyPy"'] = [
|
||||
# Earlier lxml versions are affected by
|
||||
# https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,
|
||||
# https://foss.heptapod.net/pypy/pypy/-/issues/2498,
|
||||
# which was fixed in Cython 0.26, released on 2017-06-19, and used to
|
||||
# generate the C headers of lxml release tarballs published since then, the
|
||||
# first of which was:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,17 @@
|
|||
import scrapy
|
||||
from scrapy.crawler import CrawlerProcess
|
||||
|
||||
|
||||
class NoRequestsSpider(scrapy.Spider):
|
||||
name = 'no_request'
|
||||
|
||||
def start_requests(self):
|
||||
return []
|
||||
|
||||
|
||||
process = CrawlerProcess(settings={
|
||||
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
|
||||
"ASYNCIO_EVENT_LOOP": "uvloop.Loop"
|
||||
})
|
||||
process.crawl(NoRequestsSpider)
|
||||
process.start()
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
from argparse import ArgumentParser
|
||||
|
||||
from pyftpdlib.authorizers import DummyAuthorizer
|
||||
from pyftpdlib.handlers import FTPHandler
|
||||
from pyftpdlib.servers import FTPServer
|
||||
|
||||
|
||||
def main():
|
||||
parser = ArgumentParser()
|
||||
parser.add_argument('-d', '--directory')
|
||||
args = parser.parse_args()
|
||||
|
||||
authorizer = DummyAuthorizer()
|
||||
full_permissions = 'elradfmwMT'
|
||||
authorizer.add_anonymous(args.directory, perm=full_permissions)
|
||||
handler = FTPHandler
|
||||
handler.authorizer = authorizer
|
||||
address = ('127.0.0.1', 2121)
|
||||
server = FTPServer(address, handler)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -1,32 +1,50 @@
|
|||
-----BEGIN RSA PRIVATE KEY-----
|
||||
MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt
|
||||
auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha
|
||||
QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB
|
||||
AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF
|
||||
QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol
|
||||
Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w
|
||||
T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll
|
||||
KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj
|
||||
58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc
|
||||
1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT
|
||||
9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB
|
||||
dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn
|
||||
Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA==
|
||||
-----END RSA PRIVATE KEY-----
|
||||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB
|
||||
/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT
|
||||
NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ
|
||||
u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH
|
||||
JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB
|
||||
eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls
|
||||
90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ
|
||||
epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8
|
||||
VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn
|
||||
v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT
|
||||
I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU
|
||||
oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au
|
||||
/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG
|
||||
gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn
|
||||
YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf
|
||||
dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5
|
||||
9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC
|
||||
QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu
|
||||
bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW
|
||||
CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR
|
||||
OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu
|
||||
PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK
|
||||
eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx
|
||||
BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/
|
||||
B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy
|
||||
HzPh2taMnCA0h1fNLFFBkw88
|
||||
-----END PRIVATE KEY-----
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT
|
||||
CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX
|
||||
DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt
|
||||
aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u
|
||||
YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp
|
||||
w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk
|
||||
Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB
|
||||
/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG
|
||||
CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB
|
||||
gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG
|
||||
+EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n
|
||||
MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D
|
||||
y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z
|
||||
e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1
|
||||
n95u
|
||||
MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM
|
||||
CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX
|
||||
DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt
|
||||
aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW
|
||||
ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7
|
||||
ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg
|
||||
z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI
|
||||
ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE
|
||||
6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/
|
||||
0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC
|
||||
AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD
|
||||
BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG
|
||||
CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC
|
||||
AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA
|
||||
A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd
|
||||
kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe
|
||||
gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY
|
||||
1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj
|
||||
envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj
|
||||
wiKjdLjCmJgjGAQjRw5VYJ8JI0XO
|
||||
-----END CERTIFICATE-----
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@ import json
|
|||
import os
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from shutil import rmtree
|
||||
from subprocess import Popen, PIPE
|
||||
from tempfile import mkdtemp
|
||||
from urllib.parse import urlencode
|
||||
|
||||
from OpenSSL import SSL
|
||||
|
|
@ -256,6 +259,29 @@ class MockDNSServer:
|
|||
self.proc.communicate()
|
||||
|
||||
|
||||
class MockFTPServer:
|
||||
"""Creates an FTP server on port 2121 with a default passwordless user
|
||||
(anonymous) and a temporary root path that you can read from the
|
||||
:attr:`path` attribute."""
|
||||
|
||||
def __enter__(self):
|
||||
self.path = Path(mkdtemp())
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.ftpserver', '-d', str(self.path)],
|
||||
stderr=PIPE, env=get_testenv())
|
||||
for line in self.proc.stderr:
|
||||
if b'starting FTP server' in line:
|
||||
break
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
rmtree(str(self.path))
|
||||
self.proc.kill()
|
||||
self.proc.communicate()
|
||||
|
||||
def url(self, path):
|
||||
return 'ftp://127.0.0.1:2121/' + path
|
||||
|
||||
|
||||
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
|
||||
factory = ssl.DefaultOpenSSLContextFactory(
|
||||
os.path.join(os.path.dirname(__file__), keyfile),
|
||||
|
|
@ -263,8 +289,8 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
|
|||
)
|
||||
if cipher_string:
|
||||
ctx = factory.getContext()
|
||||
# disabling TLS1.2+ because it unconditionally enables some strong ciphers
|
||||
ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2 | SSL_OP_NO_TLSv1_3)
|
||||
# disabling TLS1.3 because it unconditionally enables some strong ciphers
|
||||
ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3)
|
||||
ctx.set_cipher_list(to_bytes(cipher_string))
|
||||
return factory
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
# Tests requirements
|
||||
attrs
|
||||
dataclasses; python_version == '3.6'
|
||||
mitmproxy; python_version >= '3.6'
|
||||
mitmproxy<4.0.0; python_version < '3.6'
|
||||
mitmproxy; python_version >= '3.7'
|
||||
mitmproxy >= 4, < 5; python_version >= '3.6' and python_version < '3.7'
|
||||
mitmproxy < 4; python_version < '3.6'
|
||||
pyftpdlib
|
||||
# https://github.com/pytest-dev/pytest-twisted/issues/93
|
||||
pytest != 5.4, != 5.4.1
|
||||
pytest-azurepipelines
|
||||
|
|
@ -11,6 +13,7 @@ pytest-twisted >= 1.11
|
|||
pytest-xdist
|
||||
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
|
||||
testfixtures
|
||||
uvloop; platform_system != "Windows"
|
||||
|
||||
# optional for shell wrapper tests
|
||||
bpython
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import sys
|
|||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from itertools import chain
|
||||
from os.path import exists, join, abspath
|
||||
from os.path import exists, join, abspath, getmtime
|
||||
from pathlib import Path
|
||||
from shutil import rmtree, copytree
|
||||
from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
|
||||
|
|
@ -16,6 +16,7 @@ from tempfile import mkdtemp
|
|||
from threading import Timer
|
||||
from unittest import skipIf
|
||||
|
||||
from pytest import mark
|
||||
from twisted.trial import unittest
|
||||
|
||||
import scrapy
|
||||
|
|
@ -74,6 +75,7 @@ class ProjectTest(unittest.TestCase):
|
|||
|
||||
def kill_proc():
|
||||
p.kill()
|
||||
p.communicate()
|
||||
assert False, 'Command took too much time to complete'
|
||||
|
||||
timer = Timer(15, kill_proc)
|
||||
|
|
@ -337,8 +339,11 @@ class GenspiderCommandTest(CommandTest):
|
|||
p, out, err = self.proc('genspider', spname, 'test.com', *args)
|
||||
self.assertIn("Created spider %r using template %r in module" % (spname, tplname), out)
|
||||
self.assertTrue(exists(join(self.proj_mod_path, 'spiders', 'test_spider.py')))
|
||||
modify_time_before = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
|
||||
p, out, err = self.proc('genspider', spname, 'test.com', *args)
|
||||
self.assertIn("Spider %r already exists in module" % spname, out)
|
||||
modify_time_after = getmtime(join(self.proj_mod_path, 'spiders', 'test_spider.py'))
|
||||
self.assertEqual(modify_time_after, modify_time_before)
|
||||
|
||||
def test_template_basic(self):
|
||||
self.test_template('basic')
|
||||
|
|
@ -360,6 +365,40 @@ class GenspiderCommandTest(CommandTest):
|
|||
self.assertEqual(2, self.call('genspider', self.project_name))
|
||||
assert not exists(join(self.proj_mod_path, 'spiders', '%s.py' % self.project_name))
|
||||
|
||||
def test_same_filename_as_existing_spider(self, force=False):
|
||||
file_name = 'example'
|
||||
file_path = join(self.proj_mod_path, 'spiders', '%s.py' % file_name)
|
||||
self.assertEqual(0, self.call('genspider', file_name, 'example.com'))
|
||||
assert exists(file_path)
|
||||
|
||||
# change name of spider but not its file name
|
||||
with open(file_path, 'r+') as spider_file:
|
||||
file_data = spider_file.read()
|
||||
file_data = file_data.replace("name = \'example\'", "name = \'renamed\'")
|
||||
spider_file.seek(0)
|
||||
spider_file.write(file_data)
|
||||
spider_file.truncate()
|
||||
modify_time_before = getmtime(file_path)
|
||||
file_contents_before = file_data
|
||||
|
||||
if force:
|
||||
p, out, err = self.proc('genspider', '--force', file_name, 'example.com')
|
||||
self.assertIn("Created spider %r using template \'basic\' in module" % file_name, out)
|
||||
modify_time_after = getmtime(file_path)
|
||||
self.assertNotEqual(modify_time_after, modify_time_before)
|
||||
file_contents_after = open(file_path, 'r').read()
|
||||
self.assertNotEqual(file_contents_after, file_contents_before)
|
||||
else:
|
||||
p, out, err = self.proc('genspider', file_name, 'example.com')
|
||||
self.assertIn("%s already exists" % (file_path), out)
|
||||
modify_time_after = getmtime(file_path)
|
||||
self.assertEqual(modify_time_after, modify_time_before)
|
||||
file_contents_after = open(file_path, 'r').read()
|
||||
self.assertEqual(file_contents_after, file_contents_before)
|
||||
|
||||
def test_same_filename_as_existing_spider_force(self):
|
||||
self.test_same_filename_as_existing_spider(force=True)
|
||||
|
||||
|
||||
class GenspiderStandaloneCommandTest(ProjectTest):
|
||||
|
||||
|
|
@ -367,6 +406,34 @@ class GenspiderStandaloneCommandTest(ProjectTest):
|
|||
self.call('genspider', 'example', 'example.com')
|
||||
assert exists(join(self.temp_path, 'example.py'))
|
||||
|
||||
def test_same_name_as_existing_file(self, force=False):
|
||||
file_name = 'example'
|
||||
file_path = join(self.temp_path, file_name + '.py')
|
||||
p, out, err = self.proc('genspider', file_name, 'example.com')
|
||||
self.assertIn("Created spider %r using template \'basic\' " % file_name, out)
|
||||
assert exists(file_path)
|
||||
modify_time_before = getmtime(file_path)
|
||||
file_contents_before = open(file_path, 'r').read()
|
||||
|
||||
if force:
|
||||
# use different template to ensure contents were changed
|
||||
p, out, err = self.proc('genspider', '--force', '-t', 'crawl', file_name, 'example.com')
|
||||
self.assertIn("Created spider %r using template \'crawl\' " % file_name, out)
|
||||
modify_time_after = getmtime(file_path)
|
||||
self.assertNotEqual(modify_time_after, modify_time_before)
|
||||
file_contents_after = open(file_path, 'r').read()
|
||||
self.assertNotEqual(file_contents_after, file_contents_before)
|
||||
else:
|
||||
p, out, err = self.proc('genspider', file_name, 'example.com')
|
||||
self.assertIn("%s already exists" % join(self.temp_path, file_name + ".py"), out)
|
||||
modify_time_after = getmtime(file_path)
|
||||
self.assertEqual(modify_time_after, modify_time_before)
|
||||
file_contents_after = open(file_path, 'r').read()
|
||||
self.assertEqual(file_contents_after, file_contents_before)
|
||||
|
||||
def test_same_name_as_existing_file_force(self):
|
||||
self.test_same_name_as_existing_file(force=True)
|
||||
|
||||
|
||||
class MiscCommandsTest(CommandTest):
|
||||
|
||||
|
|
@ -504,6 +571,77 @@ class BadSpider(scrapy.Spider):
|
|||
log = self.get_log(self.debug_log_spider, args=[])
|
||||
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
|
||||
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
|
||||
def test_custom_asyncio_loop_enabled_true(self):
|
||||
log = self.get_log(self.debug_log_spider, args=[
|
||||
'-s',
|
||||
'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor',
|
||||
'-s',
|
||||
'ASYNCIO_EVENT_LOOP=uvloop.Loop',
|
||||
])
|
||||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
|
||||
# https://twistedmatrix.com/trac/ticket/9766
|
||||
@skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
|
||||
"the asyncio reactor is broken on Windows when running Python ≥ 3.8")
|
||||
def test_custom_asyncio_loop_enabled_false(self):
|
||||
log = self.get_log(self.debug_log_spider, args=[
|
||||
'-s', 'TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor'
|
||||
])
|
||||
import asyncio
|
||||
loop = asyncio.new_event_loop()
|
||||
self.assertIn("Using asyncio event loop: %s.%s" % (loop.__module__, loop.__class__.__name__), log)
|
||||
|
||||
def test_output(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
|
||||
return []
|
||||
"""
|
||||
args = ['-o', 'example.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
|
||||
|
||||
def test_overwrite_output(self):
|
||||
spider_code = """
|
||||
import json
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug(
|
||||
'FEEDS: {}'.format(
|
||||
json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
|
||||
)
|
||||
)
|
||||
return []
|
||||
"""
|
||||
args = ['-O', 'example.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
|
||||
|
||||
def test_output_and_overwrite_output(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
return []
|
||||
"""
|
||||
args = ['-o', 'example1.json', '-O', 'example2.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
|
||||
|
||||
|
||||
class BenchCommandTest(CommandTest):
|
||||
|
||||
|
|
@ -512,3 +650,79 @@ class BenchCommandTest(CommandTest):
|
|||
'-s', 'CLOSESPIDER_TIMEOUT=0.01')
|
||||
self.assertIn('INFO: Crawled', log)
|
||||
self.assertNotIn('Unhandled Error', log)
|
||||
|
||||
|
||||
class CrawlCommandTest(CommandTest):
|
||||
|
||||
def crawl(self, code, args=()):
|
||||
fname = abspath(join(self.proj_mod_path, 'spiders', 'myspider.py'))
|
||||
with open(fname, 'w') as f:
|
||||
f.write(code)
|
||||
return self.proc('crawl', 'myspider', *args)
|
||||
|
||||
def get_log(self, code, args=()):
|
||||
_, _, stderr = self.crawl(code, args=args)
|
||||
return stderr
|
||||
|
||||
def test_no_output(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug('It works!')
|
||||
return []
|
||||
"""
|
||||
log = self.get_log(spider_code)
|
||||
self.assertIn("[myspider] DEBUG: It works!", log)
|
||||
|
||||
def test_output(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
|
||||
return []
|
||||
"""
|
||||
args = ['-o', 'example.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn("[myspider] DEBUG: FEEDS: {'example.json': {'format': 'json'}}", log)
|
||||
|
||||
def test_overwrite_output(self):
|
||||
spider_code = """
|
||||
import json
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
self.logger.debug(
|
||||
'FEEDS: {}'.format(
|
||||
json.dumps(self.settings.getdict('FEEDS'), sort_keys=True)
|
||||
)
|
||||
)
|
||||
return []
|
||||
"""
|
||||
args = ['-O', 'example.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn('[myspider] DEBUG: FEEDS: {"example.json": {"format": "json", "overwrite": true}}', log)
|
||||
|
||||
def test_output_and_overwrite_output(self):
|
||||
spider_code = """
|
||||
import scrapy
|
||||
|
||||
class MySpider(scrapy.Spider):
|
||||
name = 'myspider'
|
||||
|
||||
def start_requests(self):
|
||||
return []
|
||||
"""
|
||||
args = ['-o', 'example1.json', '-O', 'example2.json']
|
||||
log = self.get_log(spider_code, args=args)
|
||||
self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
|
||||
|
|
|
|||
|
|
@ -142,7 +142,7 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
|
|||
|
||||
def test_spider_manager_verify_interface(self):
|
||||
settings = Settings({
|
||||
'SPIDER_LOADER_CLASS': 'tests.test_crawler.SpiderLoaderWithWrongInterface'
|
||||
'SPIDER_LOADER_CLASS': SpiderLoaderWithWrongInterface,
|
||||
})
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
self.assertRaises(AttributeError, CrawlerRunner, settings)
|
||||
|
|
@ -345,6 +345,14 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
|||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
|
||||
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
|
||||
def test_custom_loop_asyncio(self):
|
||||
log = self.run_script("asyncio_custom_loop.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
self.assertIn("Using asyncio event loop: uvloop.Loop", log)
|
||||
|
||||
|
||||
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ class OffDH:
|
|||
class LoadTestCase(unittest.TestCase):
|
||||
|
||||
def test_enabled_handler(self):
|
||||
handlers = {'scheme': 'tests.test_downloader_handlers.DummyDH'}
|
||||
handlers = {'scheme': DummyDH}
|
||||
crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
|
||||
dh = DownloadHandlers(crawler)
|
||||
self.assertIn('scheme', dh._schemes)
|
||||
|
|
@ -69,7 +69,7 @@ class LoadTestCase(unittest.TestCase):
|
|||
self.assertNotIn('scheme', dh._notconfigured)
|
||||
|
||||
def test_not_configured_handler(self):
|
||||
handlers = {'scheme': 'tests.test_downloader_handlers.OffDH'}
|
||||
handlers = {'scheme': OffDH}
|
||||
crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
|
||||
dh = DownloadHandlers(crawler)
|
||||
self.assertIn('scheme', dh._schemes)
|
||||
|
|
@ -87,7 +87,7 @@ class LoadTestCase(unittest.TestCase):
|
|||
self.assertIn('scheme', dh._notconfigured)
|
||||
|
||||
def test_lazy_handlers(self):
|
||||
handlers = {'scheme': 'tests.test_downloader_handlers.DummyLazyDH'}
|
||||
handlers = {'scheme': DummyLazyDH}
|
||||
crawler = get_crawler(settings_dict={'DOWNLOAD_HANDLERS': handlers})
|
||||
dh = DownloadHandlers(crawler)
|
||||
self.assertIn('scheme', dh._schemes)
|
||||
|
|
@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase):
|
|||
request = Request(self.getURL('largechunkedfile'))
|
||||
|
||||
def check(logger):
|
||||
logger.error.assert_called_once_with(mock.ANY, mock.ANY)
|
||||
logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
|
||||
|
||||
d = self.download_request(request, Spider('foo', download_maxsize=1500))
|
||||
yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from zope.interface.verify import verifyObject
|
|||
|
||||
import scrapy
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.exceptions import NotConfigured
|
||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||
from scrapy.exporters import CsvItemExporter
|
||||
from scrapy.extensions.feedexport import (
|
||||
BlockingFeedStorage,
|
||||
|
|
@ -47,7 +47,7 @@ from scrapy.utils.test import (
|
|||
mock_google_cloud_storage,
|
||||
)
|
||||
|
||||
from tests.mockserver import MockServer
|
||||
from tests.mockserver import MockFTPServer, MockServer
|
||||
|
||||
|
||||
class FileFeedStorageTest(unittest.TestCase):
|
||||
|
|
@ -76,8 +76,28 @@ class FileFeedStorageTest(unittest.TestCase):
|
|||
st = FileFeedStorage(path)
|
||||
verifyObject(IFeedStorage, st)
|
||||
|
||||
def _store(self, feed_options=None):
|
||||
path = os.path.abspath(self.mktemp())
|
||||
storage = FileFeedStorage(path, feed_options=feed_options)
|
||||
spider = scrapy.Spider("default")
|
||||
file = storage.open(spider)
|
||||
file.write(b"content")
|
||||
storage.store(file)
|
||||
return path
|
||||
|
||||
def test_append(self):
|
||||
path = self._store()
|
||||
return self._assert_stores(FileFeedStorage(path), path, b"contentcontent")
|
||||
|
||||
def test_overwrite(self):
|
||||
path = self._store({"overwrite": True})
|
||||
return self._assert_stores(
|
||||
FileFeedStorage(path, feed_options={"overwrite": True}),
|
||||
path
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _assert_stores(self, storage, path):
|
||||
def _assert_stores(self, storage, path, expected_content=b"content"):
|
||||
spider = scrapy.Spider("default")
|
||||
file = storage.open(spider)
|
||||
file.write(b"content")
|
||||
|
|
@ -85,7 +105,7 @@ class FileFeedStorageTest(unittest.TestCase):
|
|||
self.assertTrue(os.path.exists(path))
|
||||
try:
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"content")
|
||||
self.assertEqual(fp.read(), expected_content)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
|
@ -100,49 +120,74 @@ class FTPFeedStorageTest(unittest.TestCase):
|
|||
spider = TestSpider.from_crawler(crawler)
|
||||
return spider
|
||||
|
||||
def test_store(self):
|
||||
uri = os.environ.get('FEEDTEST_FTP_URI')
|
||||
path = os.environ.get('FEEDTEST_FTP_PATH')
|
||||
if not (uri and path):
|
||||
raise unittest.SkipTest("No FTP server available for testing")
|
||||
st = FTPFeedStorage(uri)
|
||||
verifyObject(IFeedStorage, st)
|
||||
return self._assert_stores(st, path)
|
||||
def _store(self, uri, content, feed_options=None, settings=None):
|
||||
crawler = get_crawler(settings_dict=settings or {})
|
||||
storage = FTPFeedStorage.from_crawler(
|
||||
crawler,
|
||||
uri,
|
||||
feed_options=feed_options,
|
||||
)
|
||||
verifyObject(IFeedStorage, storage)
|
||||
spider = self.get_test_spider()
|
||||
file = storage.open(spider)
|
||||
file.write(content)
|
||||
return storage.store(file)
|
||||
|
||||
def test_store_active_mode(self):
|
||||
uri = os.environ.get('FEEDTEST_FTP_URI')
|
||||
path = os.environ.get('FEEDTEST_FTP_PATH')
|
||||
if not (uri and path):
|
||||
raise unittest.SkipTest("No FTP server available for testing")
|
||||
use_active_mode = {'FEED_STORAGE_FTP_ACTIVE': True}
|
||||
crawler = get_crawler(settings_dict=use_active_mode)
|
||||
st = FTPFeedStorage.from_crawler(crawler, uri)
|
||||
verifyObject(IFeedStorage, st)
|
||||
return self._assert_stores(st, path)
|
||||
def _assert_stored(self, path, content):
|
||||
self.assertTrue(path.exists())
|
||||
try:
|
||||
with path.open('rb') as fp:
|
||||
self.assertEqual(fp.read(), content)
|
||||
finally:
|
||||
os.unlink(str(path))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_append(self):
|
||||
with MockFTPServer() as ftp_server:
|
||||
filename = 'file'
|
||||
url = ftp_server.url(filename)
|
||||
feed_options = {'overwrite': False}
|
||||
yield self._store(url, b"foo", feed_options=feed_options)
|
||||
yield self._store(url, b"bar", feed_options=feed_options)
|
||||
self._assert_stored(ftp_server.path / filename, b"foobar")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_overwrite(self):
|
||||
with MockFTPServer() as ftp_server:
|
||||
filename = 'file'
|
||||
url = ftp_server.url(filename)
|
||||
yield self._store(url, b"foo")
|
||||
yield self._store(url, b"bar")
|
||||
self._assert_stored(ftp_server.path / filename, b"bar")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_append_active_mode(self):
|
||||
with MockFTPServer() as ftp_server:
|
||||
settings = {'FEED_STORAGE_FTP_ACTIVE': True}
|
||||
filename = 'file'
|
||||
url = ftp_server.url(filename)
|
||||
feed_options = {'overwrite': False}
|
||||
yield self._store(url, b"foo", feed_options=feed_options, settings=settings)
|
||||
yield self._store(url, b"bar", feed_options=feed_options, settings=settings)
|
||||
self._assert_stored(ftp_server.path / filename, b"foobar")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_overwrite_active_mode(self):
|
||||
with MockFTPServer() as ftp_server:
|
||||
settings = {'FEED_STORAGE_FTP_ACTIVE': True}
|
||||
filename = 'file'
|
||||
url = ftp_server.url(filename)
|
||||
yield self._store(url, b"foo", settings=settings)
|
||||
yield self._store(url, b"bar", settings=settings)
|
||||
self._assert_stored(ftp_server.path / filename, b"bar")
|
||||
|
||||
def test_uri_auth_quote(self):
|
||||
# RFC3986: 3.2.1. User Information
|
||||
pw_quoted = quote(string.punctuation, safe='')
|
||||
st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted)
|
||||
st = FTPFeedStorage('ftp://foo:%s@example.com/some_path' % pw_quoted,
|
||||
{})
|
||||
self.assertEqual(st.password, string.punctuation)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def _assert_stores(self, storage, path):
|
||||
spider = self.get_test_spider()
|
||||
file = storage.open(spider)
|
||||
file.write(b"content")
|
||||
yield storage.store(file)
|
||||
self.assertTrue(os.path.exists(path))
|
||||
try:
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"content")
|
||||
# again, to check s3 objects are overwritten
|
||||
yield storage.store(BytesIO(b"new content"))
|
||||
with open(path, 'rb') as fp:
|
||||
self.assertEqual(fp.read(), b"new content")
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
class BlockingFeedStorageTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -182,21 +227,19 @@ class BlockingFeedStorageTest(unittest.TestCase):
|
|||
|
||||
class S3FeedStorageTest(unittest.TestCase):
|
||||
|
||||
@mock.patch('scrapy.utils.project.get_project_settings',
|
||||
new=mock.MagicMock(return_value={'AWS_ACCESS_KEY_ID': 'conf_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'conf_secret'}),
|
||||
create=True)
|
||||
def test_parse_credentials(self):
|
||||
try:
|
||||
import boto # noqa: F401
|
||||
import botocore # noqa: F401
|
||||
except ImportError:
|
||||
raise unittest.SkipTest("S3FeedStorage requires boto")
|
||||
raise unittest.SkipTest("S3FeedStorage requires botocore")
|
||||
aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
|
||||
'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
|
||||
crawler = get_crawler(settings_dict=aws_credentials)
|
||||
# Instantiate with crawler
|
||||
storage = S3FeedStorage.from_crawler(crawler,
|
||||
's3://mybucket/export.csv')
|
||||
storage = S3FeedStorage.from_crawler(
|
||||
crawler,
|
||||
's3://mybucket/export.csv',
|
||||
)
|
||||
self.assertEqual(storage.access_key, 'settings_key')
|
||||
self.assertEqual(storage.secret_key, 'settings_secret')
|
||||
# Instantiate directly
|
||||
|
|
@ -211,12 +254,6 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
aws_credentials['AWS_SECRET_ACCESS_KEY'])
|
||||
self.assertEqual(storage.access_key, 'uri_key')
|
||||
self.assertEqual(storage.secret_key, 'uri_secret')
|
||||
# Backward compatibility for initialising without settings
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
storage = S3FeedStorage('s3://mybucket/export.csv')
|
||||
self.assertEqual(storage.access_key, 'conf_key')
|
||||
self.assertEqual(storage.secret_key, 'conf_secret')
|
||||
self.assertTrue('without AWS keys' in str(w[-1].message))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_store(self):
|
||||
|
|
@ -265,7 +302,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
crawler = get_crawler(settings_dict=settings)
|
||||
storage = S3FeedStorage.from_crawler(
|
||||
crawler,
|
||||
's3://mybucket/export.csv'
|
||||
's3://mybucket/export.csv',
|
||||
)
|
||||
self.assertEqual(storage.access_key, 'access_key')
|
||||
self.assertEqual(storage.secret_key, 'secret_key')
|
||||
|
|
@ -280,7 +317,7 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
crawler = get_crawler(settings_dict=settings)
|
||||
storage = S3FeedStorage.from_crawler(
|
||||
crawler,
|
||||
's3://mybucket/export.csv'
|
||||
's3://mybucket/export.csv',
|
||||
)
|
||||
self.assertEqual(storage.access_key, 'access_key')
|
||||
self.assertEqual(storage.secret_key, 'secret_key')
|
||||
|
|
@ -381,6 +418,27 @@ class S3FeedStorageTest(unittest.TestCase):
|
|||
key.set_contents_from_file.call_args
|
||||
)
|
||||
|
||||
def test_overwrite_default(self):
|
||||
with LogCapture() as log:
|
||||
S3FeedStorage(
|
||||
's3://mybucket/export.csv',
|
||||
'access_key',
|
||||
'secret_key',
|
||||
'custom-acl'
|
||||
)
|
||||
self.assertNotIn('S3 does not support appending to files', str(log))
|
||||
|
||||
def test_overwrite_false(self):
|
||||
with LogCapture() as log:
|
||||
S3FeedStorage(
|
||||
's3://mybucket/export.csv',
|
||||
'access_key',
|
||||
'secret_key',
|
||||
'custom-acl',
|
||||
feed_options={'overwrite': False},
|
||||
)
|
||||
self.assertIn('S3 does not support appending to files', str(log))
|
||||
|
||||
|
||||
class GCSFeedStorageTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -450,12 +508,22 @@ class StdoutFeedStorageTest(unittest.TestCase):
|
|||
yield storage.store(file)
|
||||
self.assertEqual(out.getvalue(), b"content")
|
||||
|
||||
def test_overwrite_default(self):
|
||||
with LogCapture() as log:
|
||||
StdoutFeedStorage('stdout:')
|
||||
self.assertNotIn('Standard output (stdout) storage does not support overwriting', str(log))
|
||||
|
||||
def test_overwrite_true(self):
|
||||
with LogCapture() as log:
|
||||
StdoutFeedStorage('stdout:', feed_options={'overwrite': True})
|
||||
self.assertIn('Standard output (stdout) storage does not support overwriting', str(log))
|
||||
|
||||
|
||||
class FromCrawlerMixin:
|
||||
init_with_crawler = False
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
|
||||
cls.init_with_crawler = True
|
||||
return cls(*args, **kwargs)
|
||||
|
||||
|
|
@ -465,7 +533,11 @@ class FromCrawlerCsvItemExporter(CsvItemExporter, FromCrawlerMixin):
|
|||
|
||||
|
||||
class FromCrawlerFileFeedStorage(FileFeedStorage, FromCrawlerMixin):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, feed_options=None, **kwargs):
|
||||
cls.init_with_crawler = True
|
||||
return cls(*args, feed_options=feed_options, **kwargs)
|
||||
|
||||
|
||||
class DummyBlockingFeedStorage(BlockingFeedStorage):
|
||||
|
|
@ -599,8 +671,8 @@ class FeedExportTest(FeedExportTestBase):
|
|||
|
||||
FEEDS = settings.get('FEEDS') or {}
|
||||
settings['FEEDS'] = {
|
||||
printf_escape(path_to_url(file_path)): feed
|
||||
for file_path, feed in FEEDS.items()
|
||||
printf_escape(path_to_url(file_path)): feed_options
|
||||
for file_path, feed_options in FEEDS.items()
|
||||
}
|
||||
|
||||
content = {}
|
||||
|
|
@ -610,12 +682,12 @@ class FeedExportTest(FeedExportTestBase):
|
|||
spider_cls.start_urls = [s.url('/')]
|
||||
yield runner.crawl(spider_cls)
|
||||
|
||||
for file_path, feed in FEEDS.items():
|
||||
for file_path, feed_options in FEEDS.items():
|
||||
if not os.path.exists(str(file_path)):
|
||||
continue
|
||||
|
||||
with open(str(file_path), 'rb') as f:
|
||||
content[feed['format']] = f.read()
|
||||
content[feed_options['format']] = f.read()
|
||||
|
||||
finally:
|
||||
for file_path in FEEDS.keys():
|
||||
|
|
@ -773,7 +845,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
self._random_temp_filename(): {'format': 'xml'},
|
||||
self._random_temp_filename(): {'format': 'csv'},
|
||||
},
|
||||
'FEED_STORAGES': {'file': 'tests.test_feedexport.LogOnStoreFileStorage'},
|
||||
'FEED_STORAGES': {'file': LogOnStoreFileStorage},
|
||||
'FEED_STORE_EMPTY': False
|
||||
}
|
||||
|
||||
|
|
@ -1117,8 +1189,8 @@ class FeedExportTest(FeedExportTestBase):
|
|||
@defer.inlineCallbacks
|
||||
def test_init_exporters_storages_with_crawler(self):
|
||||
settings = {
|
||||
'FEED_EXPORTERS': {'csv': 'tests.test_feedexport.FromCrawlerCsvItemExporter'},
|
||||
'FEED_STORAGES': {'file': 'tests.test_feedexport.FromCrawlerFileFeedStorage'},
|
||||
'FEED_EXPORTERS': {'csv': FromCrawlerCsvItemExporter},
|
||||
'FEED_STORAGES': {'file': FromCrawlerFileFeedStorage},
|
||||
'FEEDS': {
|
||||
self._random_temp_filename(): {'format': 'csv'},
|
||||
},
|
||||
|
|
@ -1147,7 +1219,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
self._random_temp_filename(): {'format': 'xml'},
|
||||
self._random_temp_filename(): {'format': 'csv'},
|
||||
},
|
||||
'FEED_STORAGES': {'file': 'tests.test_feedexport.DummyBlockingFeedStorage'},
|
||||
'FEED_STORAGES': {'file': DummyBlockingFeedStorage},
|
||||
}
|
||||
items = [
|
||||
{'foo': 'bar1', 'baz': ''},
|
||||
|
|
@ -1168,7 +1240,7 @@ class FeedExportTest(FeedExportTestBase):
|
|||
self._random_temp_filename(): {'format': 'xml'},
|
||||
self._random_temp_filename(): {'format': 'csv'},
|
||||
},
|
||||
'FEED_STORAGES': {'file': 'tests.test_feedexport.FailingBlockingFeedStorage'},
|
||||
'FEED_STORAGES': {'file': FailingBlockingFeedStorage},
|
||||
}
|
||||
items = [
|
||||
{'foo': 'bar1', 'baz': ''},
|
||||
|
|
@ -1553,3 +1625,262 @@ class BatchDeliveriesTest(FeedExportTestBase):
|
|||
content = json.loads(content.decode('utf-8'))
|
||||
expected_batch, items = items[:batch_size], items[batch_size:]
|
||||
self.assertEqual(expected_batch, content)
|
||||
|
||||
|
||||
class FeedExportInitTest(unittest.TestCase):
|
||||
|
||||
def test_unsupported_storage(self):
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
'unsupported://uri': {},
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
with self.assertRaises(NotConfigured):
|
||||
FeedExporter.from_crawler(crawler)
|
||||
|
||||
def test_unsupported_format(self):
|
||||
settings = {
|
||||
'FEEDS': {
|
||||
'file://path': {
|
||||
'format': 'unsupported_format',
|
||||
},
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
with self.assertRaises(NotConfigured):
|
||||
FeedExporter.from_crawler(crawler)
|
||||
|
||||
|
||||
class StdoutFeedStorageWithoutFeedOptions(StdoutFeedStorage):
|
||||
|
||||
def __init__(self, uri):
|
||||
super().__init__(uri)
|
||||
|
||||
|
||||
class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
'FEED_URI': 'file:///tmp/foobar',
|
||||
'FEED_STORAGES': {
|
||||
'file': StdoutFeedStorageWithoutFeedOptions
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"StdoutFeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove "
|
||||
"this warning. This parameter will become mandatory "
|
||||
"in a future version of Scrapy."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
|
||||
|
||||
def __init__(self, uri):
|
||||
super().__init__(uri)
|
||||
|
||||
|
||||
class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
'FEED_URI': 'file:///tmp/foobar',
|
||||
'FEED_STORAGES': {
|
||||
'file': FileFeedStorageWithoutFeedOptions
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"FileFeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove "
|
||||
"this warning. This parameter will become mandatory "
|
||||
"in a future version of Scrapy."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
|
||||
|
||||
def __init__(self, uri, access_key, secret_key, acl):
|
||||
super().__init__(uri, access_key, secret_key, acl)
|
||||
|
||||
|
||||
class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return super().from_crawler(crawler, uri)
|
||||
|
||||
|
||||
class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
'FEED_URI': 'file:///tmp/foobar',
|
||||
'FEED_STORAGES': {
|
||||
'file': S3FeedStorageWithoutFeedOptions
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"S3FeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove "
|
||||
"this warning. This parameter will become mandatory "
|
||||
"in a future version of Scrapy."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def test_from_crawler(self):
|
||||
settings_dict = {
|
||||
'FEED_URI': 'file:///tmp/foobar',
|
||||
'FEED_STORAGES': {
|
||||
'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
|
||||
"does not support the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove "
|
||||
"this warning. This parameter will become mandatory "
|
||||
"in a future version of Scrapy."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
|
||||
|
||||
def __init__(self, uri, use_active_mode=False):
|
||||
super().__init__(uri)
|
||||
|
||||
|
||||
class FTPFeedStorageWithoutFeedOptionsWithFromCrawler(FTPFeedStorage):
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, uri):
|
||||
return super().from_crawler(crawler, uri)
|
||||
|
||||
|
||||
class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
|
||||
"""Make sure that any feed exporter created by users before the
|
||||
introduction of the ``feed_options`` parameter continues to work as
|
||||
expected, and simply issues a warning."""
|
||||
|
||||
maxDiff = None
|
||||
|
||||
def test_init(self):
|
||||
settings_dict = {
|
||||
'FEED_URI': 'file:///tmp/foobar',
|
||||
'FEED_STORAGES': {
|
||||
'file': FTPFeedStorageWithoutFeedOptions
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"FTPFeedStorageWithoutFeedOptions does not support "
|
||||
"the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove "
|
||||
"this warning. This parameter will become mandatory "
|
||||
"in a future version of Scrapy."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def test_from_crawler(self):
|
||||
settings_dict = {
|
||||
'FEED_URI': 'file:///tmp/foobar',
|
||||
'FEED_STORAGES': {
|
||||
'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
|
||||
},
|
||||
}
|
||||
crawler = get_crawler(settings_dict=settings_dict)
|
||||
feed_exporter = FeedExporter.from_crawler(crawler)
|
||||
spider = scrapy.Spider("default")
|
||||
spider.crawler = crawler
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
feed_exporter.open_spider(spider)
|
||||
messages = tuple(str(item.message) for item in w
|
||||
if item.category is ScrapyDeprecationWarning)
|
||||
self.assertEqual(
|
||||
messages,
|
||||
(
|
||||
(
|
||||
"FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
|
||||
"does not support the 'feed_options' keyword argument. Add a "
|
||||
"'feed_options' parameter to its signature to remove "
|
||||
"this warning. This parameter will become mandatory "
|
||||
"in a future version of Scrapy."
|
||||
),
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ class TestMiddlewareManager(MiddlewareManager):
|
|||
|
||||
@classmethod
|
||||
def _get_mwlist_from_settings(cls, settings):
|
||||
return ['tests.test_middleware.%s' % x for x in ['M1', 'MOff', 'M3']]
|
||||
return [M1, MOff, M3]
|
||||
|
||||
def _add_middleware(self, mw):
|
||||
super()._add_middleware(mw)
|
||||
|
|
|
|||
|
|
@ -161,6 +161,19 @@ class FilesPipelineTestCase(unittest.TestCase):
|
|||
for p in patchers:
|
||||
p.stop()
|
||||
|
||||
def test_file_path_from_item(self):
|
||||
"""
|
||||
Custom file path based on item data, overriding default implementation
|
||||
"""
|
||||
class CustomFilesPipeline(FilesPipeline):
|
||||
def file_path(self, request, response=None, info=None, item=None):
|
||||
return 'full/%s' % item.get('path')
|
||||
|
||||
file_path = CustomFilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir})).file_path
|
||||
item = dict(path='path-to-store-file')
|
||||
request = Request("http://example.com")
|
||||
self.assertEqual(file_path(request, item=item), 'full/path-to-store-file')
|
||||
|
||||
|
||||
class FilesPipelineTestCaseFieldsMixin:
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@ from twisted.internet.defer import Deferred, inlineCallbacks
|
|||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.deprecate import ScrapyDeprecationWarning
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.pipelines.images import ImagesPipeline
|
||||
from scrapy.pipelines.media import MediaPipeline
|
||||
from scrapy.pipelines.files import FileException
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
|
@ -169,7 +171,7 @@ class MockedMediaPipeline(MediaPipeline):
|
|||
self._mockcalled.append('download')
|
||||
return super().download(request, info)
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
def media_to_download(self, request, info, *, item=None):
|
||||
self._mockcalled.append('media_to_download')
|
||||
if 'result' in request.meta:
|
||||
return request.meta.get('result')
|
||||
|
|
@ -179,7 +181,7 @@ class MockedMediaPipeline(MediaPipeline):
|
|||
self._mockcalled.append('get_media_requests')
|
||||
return item.get('requests')
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
def media_downloaded(self, response, request, info, *, item=None):
|
||||
self._mockcalled.append('media_downloaded')
|
||||
return super().media_downloaded(response, request, info)
|
||||
|
||||
|
|
@ -335,6 +337,123 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
['get_media_requests', 'media_to_download', 'item_completed'])
|
||||
|
||||
|
||||
class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
|
||||
self._mockcalled = []
|
||||
|
||||
def get_media_requests(self, item, info):
|
||||
item_url = item['image_urls'][0]
|
||||
return Request(
|
||||
item_url,
|
||||
meta={'response': Response(item_url, status=200, body=b'data')}
|
||||
)
|
||||
|
||||
def inc_stats(self, *args, **kwargs):
|
||||
return True
|
||||
|
||||
def media_to_download(self, request, info):
|
||||
self._mockcalled.append('media_to_download')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
|
||||
|
||||
def media_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('media_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
|
||||
|
||||
def file_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('file_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
|
||||
|
||||
def file_path(self, request, response=None, info=None):
|
||||
self._mockcalled.append('file_path')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
|
||||
|
||||
def get_images(self, response, request, info):
|
||||
self._mockcalled.append('get_images')
|
||||
return []
|
||||
|
||||
def image_downloaded(self, response, request, info):
|
||||
self._mockcalled.append('image_downloaded')
|
||||
return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
|
||||
|
||||
|
||||
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
|
||||
self.pipe.open_spider(None)
|
||||
self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
|
||||
|
||||
def _assert_method_called_with_warnings(self, method, message, warnings):
|
||||
self.assertIn(method, self.pipe._mockcalled)
|
||||
warningShown = False
|
||||
for warning in warnings:
|
||||
if warning['message'] == message and warning['category'] == ScrapyDeprecationWarning:
|
||||
warningShown = True
|
||||
self.assertTrue(warningShown)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_media_to_download_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'media_to_download(self, request, info) is deprecated, '
|
||||
'please use media_to_download(self, request, info, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('media_to_download', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_media_downloaded_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'media_downloaded(self, response, request, info) is deprecated, '
|
||||
'please use media_downloaded(self, response, request, info, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('media_downloaded', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_file_downloaded_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'file_downloaded(self, response, request, info) is deprecated, '
|
||||
'please use file_downloaded(self, response, request, info, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('file_downloaded', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_file_path_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'file_path(self, request, response=None, info=None) is deprecated, '
|
||||
'please use file_path(self, request, response=None, info=None, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('file_path', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_get_images_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'get_images(self, response, request, info) is deprecated, '
|
||||
'please use get_images(self, response, request, info, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('get_images', message, warnings)
|
||||
|
||||
@inlineCallbacks
|
||||
def test_image_downloaded_called(self):
|
||||
yield self.pipe.process_item(self.item, None)
|
||||
warnings = self.flushWarnings([MediaPipeline._compatible])
|
||||
message = (
|
||||
'image_downloaded(self, response, request, info) is deprecated, '
|
||||
'please use image_downloaded(self, response, request, info, *, item=None)'
|
||||
)
|
||||
self._assert_method_called_with_warnings('image_downloaded', message, warnings)
|
||||
|
||||
|
||||
class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
|
||||
|
||||
def _assert_request_no3xx(self, pipeline_class, settings):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,202 @@
|
|||
from twisted.internet import defer
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy import Request, signals
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.http.response import Response
|
||||
|
||||
from testfixtures import LogCapture
|
||||
|
||||
from tests.mockserver import MockServer
|
||||
from tests.spiders import SingleRequestSpider
|
||||
|
||||
|
||||
OVERRIDEN_URL = "https://example.org"
|
||||
|
||||
|
||||
class ProcessResponseMiddleware:
|
||||
def process_response(self, request, response, spider):
|
||||
return response.replace(request=Request(OVERRIDEN_URL))
|
||||
|
||||
|
||||
class RaiseExceptionRequestMiddleware:
|
||||
def process_request(self, request, spider):
|
||||
1 / 0
|
||||
return request
|
||||
|
||||
|
||||
class CatchExceptionOverrideRequestMiddleware:
|
||||
def process_exception(self, request, exception, spider):
|
||||
return Response(
|
||||
url="http://localhost/",
|
||||
body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
|
||||
request=Request(OVERRIDEN_URL),
|
||||
)
|
||||
|
||||
|
||||
class CatchExceptionDoNotOverrideRequestMiddleware:
|
||||
def process_exception(self, request, exception, spider):
|
||||
return Response(
|
||||
url="http://localhost/",
|
||||
body=b"Caught " + exception.__class__.__name__.encode("utf-8"),
|
||||
)
|
||||
|
||||
|
||||
class AlternativeCallbacksSpider(SingleRequestSpider):
|
||||
name = "alternative_callbacks_spider"
|
||||
|
||||
def alt_callback(self, response, foo=None):
|
||||
self.logger.info("alt_callback was invoked with foo=%s", foo)
|
||||
|
||||
|
||||
class AlternativeCallbacksMiddleware:
|
||||
def process_response(self, request, response, spider):
|
||||
new_request = request.replace(
|
||||
url=OVERRIDEN_URL,
|
||||
callback=spider.alt_callback,
|
||||
cb_kwargs={"foo": "bar"},
|
||||
)
|
||||
return response.replace(request=new_request)
|
||||
|
||||
|
||||
class CrawlTestCase(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.mockserver = MockServer()
|
||||
self.mockserver.__enter__()
|
||||
|
||||
def tearDown(self):
|
||||
self.mockserver.__exit__(None, None, None)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_200(self):
|
||||
url = self.mockserver.url("/status?n=200")
|
||||
crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
self.assertEqual(response.request.url, url)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_error(self):
|
||||
for status in ("404", "500"):
|
||||
url = self.mockserver.url("/status?n={}".format(status))
|
||||
crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
failure = crawler.spider.meta["failure"]
|
||||
response = failure.value.response
|
||||
self.assertEqual(failure.request.url, url)
|
||||
self.assertEqual(response.request.url, url)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_downloader_middleware_raise_exception(self):
|
||||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".RaiseExceptionRequestMiddleware": 590,
|
||||
},
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
failure = crawler.spider.meta["failure"]
|
||||
self.assertEqual(failure.request.url, url)
|
||||
self.assertIsInstance(failure.value, ZeroDivisionError)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_downloader_middleware_override_request_in_process_response(self):
|
||||
"""
|
||||
Downloader middleware which returns a response with an specific 'request' attribute.
|
||||
|
||||
* The spider callback should receive the overriden response.request
|
||||
* Handlers listening to the response_received signal should receive the overriden response.request
|
||||
* The "crawled" log message should show the overriden response.request
|
||||
"""
|
||||
signal_params = {}
|
||||
|
||||
def signal_handler(response, request, spider):
|
||||
signal_params["response"] = response
|
||||
signal_params["request"] = request
|
||||
|
||||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".ProcessResponseMiddleware": 595,
|
||||
}
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
crawler.signals.connect(signal_handler, signal=signals.response_received)
|
||||
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
self.assertEqual(response.request.url, OVERRIDEN_URL)
|
||||
|
||||
self.assertEqual(signal_params["response"].url, url)
|
||||
self.assertEqual(signal_params["request"].url, OVERRIDEN_URL)
|
||||
|
||||
log.check_present(
|
||||
("scrapy.core.engine", "DEBUG", "Crawled (200) <GET {}> (referer: None)".format(OVERRIDEN_URL)),
|
||||
)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_downloader_middleware_override_in_process_exception(self):
|
||||
"""
|
||||
An exception is raised but caught by the next middleware, which
|
||||
returns a Response with a specific 'request' attribute.
|
||||
|
||||
The spider callback should receive the overriden response.request
|
||||
"""
|
||||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".RaiseExceptionRequestMiddleware": 590,
|
||||
__name__ + ".CatchExceptionOverrideRequestMiddleware": 595,
|
||||
},
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
self.assertEqual(response.body, b"Caught ZeroDivisionError")
|
||||
self.assertEqual(response.request.url, OVERRIDEN_URL)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_downloader_middleware_do_not_override_in_process_exception(self):
|
||||
"""
|
||||
An exception is raised but caught by the next middleware, which
|
||||
returns a Response without a specific 'request' attribute.
|
||||
|
||||
The spider callback should receive the original response.request
|
||||
"""
|
||||
url = self.mockserver.url("/status?n=200")
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".RaiseExceptionRequestMiddleware": 590,
|
||||
__name__ + ".CatchExceptionDoNotOverrideRequestMiddleware": 595,
|
||||
},
|
||||
})
|
||||
crawler = runner.create_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
self.assertEqual(response.body, b"Caught ZeroDivisionError")
|
||||
self.assertEqual(response.request.url, url)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_downloader_middleware_alternative_callback(self):
|
||||
"""
|
||||
Downloader middleware which returns a response with a
|
||||
specific 'request' attribute, with an alternative callback
|
||||
"""
|
||||
runner = CrawlerRunner(settings={
|
||||
"DOWNLOADER_MIDDLEWARES": {
|
||||
__name__ + ".AlternativeCallbacksMiddleware": 595,
|
||||
}
|
||||
})
|
||||
crawler = runner.create_crawler(AlternativeCallbacksSpider)
|
||||
|
||||
with LogCapture() as log:
|
||||
url = self.mockserver.url("/status?n=200")
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
|
||||
log.check_present(
|
||||
("alternative_callbacks_spider", "INFO", "alt_callback was invoked with foo=bar"),
|
||||
)
|
||||
|
|
@ -385,6 +385,38 @@ class SettingsTest(unittest.TestCase):
|
|||
self.assertIn('key', mydict)
|
||||
self.assertEqual(mydict['key'], 'val')
|
||||
|
||||
def test_passing_objects_as_values(self):
|
||||
from scrapy.core.downloader.handlers.file import FileDownloadHandler
|
||||
from scrapy.utils.misc import create_instance
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
class TestPipeline():
|
||||
def process_item(self, i, s):
|
||||
return i
|
||||
|
||||
settings = Settings({
|
||||
'ITEM_PIPELINES': {
|
||||
TestPipeline: 800,
|
||||
},
|
||||
'DOWNLOAD_HANDLERS': {
|
||||
'ftp': FileDownloadHandler,
|
||||
},
|
||||
})
|
||||
|
||||
self.assertIn('ITEM_PIPELINES', settings.attributes)
|
||||
|
||||
mypipeline, priority = settings.getdict('ITEM_PIPELINES').popitem()
|
||||
self.assertEqual(priority, 800)
|
||||
self.assertEqual(mypipeline, TestPipeline)
|
||||
self.assertIsInstance(mypipeline(), TestPipeline)
|
||||
self.assertEqual(mypipeline().process_item('item', None), 'item')
|
||||
|
||||
myhandler = settings.getdict('DOWNLOAD_HANDLERS').pop('ftp')
|
||||
self.assertEqual(myhandler, FileDownloadHandler)
|
||||
myhandler_instance = create_instance(myhandler, None, get_crawler())
|
||||
self.assertIsInstance(myhandler_instance, FileDownloadHandler)
|
||||
self.assertTrue(hasattr(myhandler_instance, 'download_request'))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -385,7 +385,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
|
|||
|
||||
|
||||
class TestSettingsCustomPolicy(TestRefererMiddleware):
|
||||
settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
|
||||
settings = {'REFERRER_POLICY': CustomPythonOrgPolicy}
|
||||
scenarii = [
|
||||
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
|
||||
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
|
||||
|
|
|
|||
|
|
@ -141,6 +141,22 @@ class FeedExportConfigTestCase(unittest.TestCase):
|
|||
feed_process_params_from_cli(settings, ['-:pickle'])
|
||||
)
|
||||
|
||||
def test_feed_export_config_overwrite(self):
|
||||
settings = Settings()
|
||||
self.assertEqual(
|
||||
{'output.json': {'format': 'json', 'overwrite': True}},
|
||||
feed_process_params_from_cli(settings, [], None, ['output.json'])
|
||||
)
|
||||
|
||||
def test_output_and_overwrite_output(self):
|
||||
with self.assertRaises(UsageError):
|
||||
feed_process_params_from_cli(
|
||||
Settings(),
|
||||
['output1.json'],
|
||||
None,
|
||||
['output2.json'],
|
||||
)
|
||||
|
||||
def test_feed_complete_default_values_from_settings_empty(self):
|
||||
feed = {}
|
||||
settings = Settings({
|
||||
|
|
|
|||
|
|
@ -7,9 +7,6 @@ from scrapy.http import XmlResponse, TextResponse, Response
|
|||
from tests import get_testdata
|
||||
|
||||
|
||||
FOOBAR_NL = "foo{}bar".format(os.linesep)
|
||||
|
||||
|
||||
class XmliterTestCase(unittest.TestCase):
|
||||
|
||||
xmliter = staticmethod(xmliter)
|
||||
|
|
@ -267,7 +264,7 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual(result,
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '3', 'name': 'multi', 'value': "foo\nbar"},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
# explicit type check cuz' we no like stinkin' autocasting! yarrr
|
||||
|
|
@ -283,7 +280,7 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual([row for row in csv],
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '3', 'name': 'multi', 'value': "foo\nbar"},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_quotechar(self):
|
||||
|
|
@ -296,7 +293,7 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual([row for row in csv1],
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '3', 'name': 'multi', 'value': "foo\nbar"},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
response2 = TextResponse(url="http://example.com/", body=body2)
|
||||
|
|
@ -305,7 +302,7 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual([row for row in csv2],
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '3', 'name': 'multi', 'value': "foo\nbar"},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_wrong_quotechar(self):
|
||||
|
|
@ -327,7 +324,7 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual([row for row in csv],
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '3', 'name': 'multi', 'value': "foo\nbar"},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_headers(self):
|
||||
|
|
@ -353,7 +350,7 @@ class UtilsCsvTestCase(unittest.TestCase):
|
|||
self.assertEqual([row for row in csv],
|
||||
[{'id': '1', 'name': 'alpha', 'value': 'foobar'},
|
||||
{'id': '2', 'name': 'unicode', 'value': '\xfan\xedc\xf3d\xe9\u203d'},
|
||||
{'id': '3', 'name': 'multi', 'value': FOOBAR_NL},
|
||||
{'id': '3', 'name': 'multi', 'value': "foo\nbar"},
|
||||
{'id': '4', 'name': 'empty', 'value': ''}])
|
||||
|
||||
def test_csviter_exception(self):
|
||||
|
|
|
|||
|
|
@ -12,11 +12,22 @@ __doctests__ = ['scrapy.utils.misc']
|
|||
|
||||
class UtilsMiscTestCase(unittest.TestCase):
|
||||
|
||||
def test_load_object(self):
|
||||
def test_load_object_class(self):
|
||||
obj = load_object(Field)
|
||||
self.assertIs(obj, Field)
|
||||
obj = load_object('scrapy.item.Field')
|
||||
self.assertIs(obj, Field)
|
||||
|
||||
def test_load_object_function(self):
|
||||
obj = load_object(load_object)
|
||||
self.assertIs(obj, load_object)
|
||||
obj = load_object('scrapy.utils.misc.load_object')
|
||||
assert obj is load_object
|
||||
self.assertIs(obj, load_object)
|
||||
|
||||
def test_load_object_exceptions(self):
|
||||
self.assertRaises(ImportError, load_object, 'nomodule999.mod.function')
|
||||
self.assertRaises(NameError, load_object, 'scrapy.utils.misc.load_object999')
|
||||
self.assertRaises(TypeError, load_object, dict())
|
||||
|
||||
def test_walk_modules(self):
|
||||
mods = walk_modules('tests.test_utils_misc.test_walk_modules')
|
||||
|
|
|
|||
|
|
@ -29,9 +29,28 @@ class UtilsMiscPy3TestCase(unittest.TestCase):
|
|||
yield 1
|
||||
yield from g()
|
||||
|
||||
def m():
|
||||
yield 1
|
||||
|
||||
def helper():
|
||||
return 0
|
||||
|
||||
yield helper()
|
||||
|
||||
def n():
|
||||
yield 1
|
||||
|
||||
def helper():
|
||||
return 0
|
||||
|
||||
yield helper()
|
||||
return 2
|
||||
|
||||
assert is_generator_with_return_value(f)
|
||||
assert is_generator_with_return_value(g)
|
||||
assert not is_generator_with_return_value(h)
|
||||
assert not is_generator_with_return_value(i)
|
||||
assert not is_generator_with_return_value(j)
|
||||
assert not is_generator_with_return_value(k) # not recursive
|
||||
assert not is_generator_with_return_value(m)
|
||||
assert is_generator_with_return_value(n)
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@ import gc
|
|||
import operator
|
||||
import platform
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from itertools import count
|
||||
from sys import version_info
|
||||
from warnings import catch_warnings
|
||||
|
||||
from scrapy.utils.python import (
|
||||
|
|
@ -179,6 +179,9 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
def f2(a, b=None, c=None):
|
||||
pass
|
||||
|
||||
def f3(a, b=None, *, c=None):
|
||||
pass
|
||||
|
||||
class A:
|
||||
def __init__(self, a, b, c):
|
||||
pass
|
||||
|
|
@ -199,6 +202,7 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
|
||||
self.assertEqual(get_func_args(f1), ['a', 'b', 'c'])
|
||||
self.assertEqual(get_func_args(f2), ['a', 'b', 'c'])
|
||||
self.assertEqual(get_func_args(f3), ['a', 'b', 'c'])
|
||||
self.assertEqual(get_func_args(A), ['a', 'b', 'c'])
|
||||
self.assertEqual(get_func_args(a.method), ['a', 'b', 'c'])
|
||||
self.assertEqual(get_func_args(partial_f1), ['b', 'c'])
|
||||
|
|
@ -212,15 +216,15 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
self.assertEqual(get_func_args(str.split), [])
|
||||
self.assertEqual(get_func_args(" ".join), [])
|
||||
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
|
||||
else:
|
||||
self.assertEqual(
|
||||
get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
|
||||
self.assertEqual(
|
||||
get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
|
||||
if version_info < (3, 6):
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
|
||||
else:
|
||||
elif platform.python_implementation() == 'PyPy':
|
||||
self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
|
||||
self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
|
||||
|
||||
build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y')
|
||||
if build_date >= datetime(2020, 4, 7): # PyPy 3.6-v7.3.1
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
|
||||
else:
|
||||
self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
|
||||
|
||||
def test_without_none_values(self):
|
||||
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
|
||||
|
|
|
|||
|
|
@ -102,6 +102,12 @@ class RequestSerializationTest(unittest.TestCase):
|
|||
errback=self.spider.handle_error)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
||||
def test_delegated_callback_serialization(self):
|
||||
r = Request("http://www.example.com",
|
||||
callback=self.spider.delegated_callback,
|
||||
errback=self.spider.handle_error)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
||||
def test_unserializable_callback1(self):
|
||||
r = Request("http://www.example.com", callback=lambda x: x)
|
||||
self.assertRaises(ValueError, request_to_dict, r)
|
||||
|
|
@ -132,6 +138,11 @@ class TestSpiderMixin:
|
|||
pass
|
||||
|
||||
|
||||
class TestSpiderDelegation:
|
||||
def delegated_callback(self, response):
|
||||
pass
|
||||
|
||||
|
||||
def parse_item(response):
|
||||
pass
|
||||
|
||||
|
|
@ -155,6 +166,9 @@ class TestSpider(Spider, TestSpiderMixin):
|
|||
__parse_item_reference = private_parse_item
|
||||
__handle_error_reference = private_handle_error
|
||||
|
||||
def __init__(self):
|
||||
self.delegated_callback = TestSpiderDelegation().delegated_callback
|
||||
|
||||
def parse_item(self, response):
|
||||
pass
|
||||
|
||||
|
|
|
|||
8
tox.ini
8
tox.ini
|
|
@ -13,8 +13,8 @@ deps =
|
|||
-rtests/requirements-py3.txt
|
||||
# Extras
|
||||
boto3>=1.13.0
|
||||
botocore>=1.3.23
|
||||
Pillow>=3.4.2
|
||||
botocore>=1.4.87
|
||||
Pillow>=4.0.0
|
||||
passenv =
|
||||
S3_TEST_FILE_URI
|
||||
AWS_ACCESS_KEY_ID
|
||||
|
|
@ -76,9 +76,9 @@ deps =
|
|||
zope.interface==4.1.3
|
||||
-rtests/requirements-py3.txt
|
||||
# Extras
|
||||
botocore==1.3.23
|
||||
botocore==1.4.87
|
||||
google-cloud-storage==1.29.0
|
||||
Pillow==3.4.2
|
||||
Pillow==4.0.0
|
||||
|
||||
[testenv:pinned]
|
||||
deps =
|
||||
|
|
|
|||
Loading…
Reference in New Issue