RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into bytes_received_signal

This commit is contained in:
Eugenio Lacuesta 2020-05-01 16:11:32 -03:00
parent c4a5e3f0da b6eae22b38
commit 4d625afda5
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
75 changed files with 1102 additions and 707 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.0.0
current_version = 2.1.0
commit = True
tag = True
tag_name = {new_version}

22
.travis.yml
View File

@ -11,25 +11,23 @@ matrix:
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=pypy3
- env: TOXENV=py35
python: 3.5
- python: 3.5
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py35-asyncio
- env: TOXENV=asyncio
python: 3.5.2
- env: TOXENV=py36
python: 3.6
- env: TOXENV=py37
python: 3.7
- env: TOXENV=py38
- python: 3.6
- python: 3.7
- env: PYPI_RELEASE_JOB=true
python: 3.8
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=py38-asyncio
- env: TOXENV=asyncio
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
@ -62,4 +60,4 @@ deploy:
on:
tags: true
repo: scrapy/scrapy
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

2
conftest.py
View File

@ -12,6 +12,8 @@ collect_ignore = [
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]

7
docs/conf.py
View File

@ -295,3 +295,10 @@ intersphinx_mapping = {
# ------------------------------------
hoverxref_auto_ref = True
hoverxref_role_types = {
"class": "tooltip",
"confval": "tooltip",
"hoverxref": "tooltip",
"mod": "tooltip",
"ref": "tooltip",
}

15
docs/intro/tutorial.rst
View File

@ -25,16 +25,16 @@ Scrapy.
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
If you're new to programming and want to start with Python, the following books
may be useful to you:
may be useful to you:
* `Automate the Boring Stuff With Python`_
* `How To Think Like a Computer Scientist`_
* `How To Think Like a Computer Scientist`_
* `Learn Python 3 The Hard Way`_
* `Learn Python 3 The Hard Way`_
You can also take a look at `this list of Python resources for non-programmers`_,
as well as the `suggested resources in the learnpython-subreddit`_.
as well as the `suggested resources in the learnpython-subreddit`_.
.. _Python: https://www.python.org/
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents::
__init__.py
items.py # project items definition file
middlewares.py # project middlewares file
pipelines.py # project pipelines file
@ -287,8 +287,8 @@ to be scraped, you can at least get **some** data.
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
expressions`_:
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
:doc:`regular expressions <library/re>`:
>>> response.css('title::text').re(r'Quotes.*')
['Quotes to Scrape']
@ -305,7 +305,6 @@ with a selector (see :ref:`topics-developer-tools`).
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
visually selected elements, which works in many browsers.
.. _regular expressions: https://docs.python.org/3/library/re.html
.. _Selector Gadget: https://selectorgadget.com/

147
docs/news.rst
View File

@ -3,6 +3,153 @@
Release notes
=============
.. _release-2.1.0:
Scrapy 2.1.0 (2020-04-24)
-------------------------
Highlights:
* New :setting:`FEEDS` setting to export to multiple feeds
* New :attr:`Response.ip_address <scrapy.http.Response.ip_address>` attribute
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* :exc:`AssertionError` exceptions triggered by :ref:`assert <assert>`
statements have been replaced by new exception types, to support running
Python in optimized mode (see :option:`-O`) without changing Scrapy’s
behavior in any unexpected ways.
If you catch an :exc:`AssertionError` exception from Scrapy, update your
code to catch the corresponding new exception.
(:issue:`4440`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use
:setting:`SCHEDULER_DEBUG` instead (:issue:`4385`)
* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use
:setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`)
* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware`
middleware has been removed, including the entire
:class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers
work out of the box (:issue:`4431`)
* The ``spiders`` property has been removed from
:class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` or instantiate
:setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`)
* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode``
classes have been removed from :mod:`scrapy.utils.datatypes`
(:issue:`4400`)
Deprecations
~~~~~~~~~~~~
* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in
favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`,
:issue:`4507`)
New features
~~~~~~~~~~~~
* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds
with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The :command:`crawl` and :command:`runspider` commands now support multiple
``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The :command:`crawl` and :command:`runspider` commands now support
specifying an output format by appending ``:<format>`` to the output file
(:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The new :attr:`Response.ip_address <scrapy.http.Response.ip_address>`
attribute gives access to the IP address that originated a response
(:issue:`3903`, :issue:`3940`)
* A warning is now issued when a value in
:attr:`~scrapy.spiders.Spider.allowed_domains` includes a port
(:issue:`50`, :issue:`3198`, :issue:`4413`)
* Zsh completion now excludes used option aliases from the completion list
(:issue:`4438`)
Bug fixes
~~~~~~~~~
* :ref:`Request serialization <request-serialization>` no longer breaks for
callbacks that are spider attributes which are assigned a function with a
different name (:issue:`4500`)
* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer
cause a :exc:`TypeError` exception (:issue:`4410`)
* Zsh completion no longer allows options after arguments (:issue:`4438`)
* zope.interface 5.0.0 and later versions are now supported
(:issue:`4447`, :issue:`4448`)
* :meth:`Spider.make_requests_from_url
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
1.4.0, now issues a warning when used (:issue:`4412`)
Documentation
~~~~~~~~~~~~~
* Improved the documentation about signals that allow their handlers to
return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`,
:issue:`4390`)
* Our PyPI entry now includes links for our documentation, our source code
repository and our issue tracker (:issue:`4456`)
* Covered the `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_
service in the documentation (:issue:`4206`, :issue:`4455`)
* Removed references to the Guppy library, which only works in Python 2
(:issue:`4285`, :issue:`4343`)
* Extended use of InterSphinx to link to Python 3 documentation
(:issue:`4444`, :issue:`4445`)
* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`,
:issue:`4496`, :issue:`4503`)
Quality assurance
~~~~~~~~~~~~~~~~~
* Removed warnings about using old, removed settings (:issue:`4404`)
* Removed a warning about importing
:class:`~twisted.internet.testing.StringTransport` from
``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`)
* Removed outdated Debian package build files (:issue:`4384`)
* Removed :class:`object` usage as a base class (:issue:`4430`)
* Removed code that added support for old versions of Twisted that we no
longer support (:issue:`4472`)
* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`,
:issue:`4481`)
* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`,
:issue:`4446`, :issue:`4489`)
.. _release-2.0.1:
Scrapy 2.0.1 (2020-03-18)

8
docs/requirements.txt
View File

@ -1,4 +1,4 @@
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page
sphinx_rtd_theme
Sphinx>=3.0
sphinx-hoverxref>=0.2b1
sphinx-notfound-page>=0.4
sphinx_rtd_theme>=0.4

4
docs/topics/contracts.rst
View File

@ -136,7 +136,7 @@ Detecting check runs
====================
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
set to the ``true`` string. You can use `os.environ`_ to perform any change to
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
your spiders or your settings when ``scrapy check`` is used::
import os
@ -148,5 +148,3 @@ your spiders or your settings when ``scrapy check`` is used::
def __init__(self):
if os.environ.get('SCRAPY_CHECK'):
pass # Do some scraper adjustments when a check is running
.. _os.environ: https://docs.python.org/3/library/os.html#os.environ

9
docs/topics/coroutines.rst
View File

@ -7,10 +7,6 @@ Coroutines
Scrapy has :ref:`partial support <coroutine-support>` for the
:ref:`coroutine syntax <async>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related API and behavior changes without a
deprecation period or warning.
.. _coroutine-support:
Supported callables
@ -76,8 +72,8 @@ becomes::
Coroutines may be used to call asynchronous code. This includes other
coroutines, functions that return Deferreds and functions that return
`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
many useful Python libraries providing such code::
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
This means you can use many useful Python libraries providing such code::
class MySpider(Spider):
# ...
@ -107,4 +103,3 @@ Common use cases for asynchronous code include:
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
.. _aio-libs: https://github.com/aio-libs
.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable

3
docs/topics/developer-tools.rst
View File

@ -292,6 +292,9 @@ Alternatively, if you want to know the arguments needed to recreate that
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
function to get a dictionary with the equivalent arguments.
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
As you can see, with a few inspections in the `Network`-tool we
were able to easily replicate the dynamic requests of the scrolling
functionality of the page. Crawling dynamic pages can be quite

9
docs/topics/downloader-middleware.rst
View File

@ -739,7 +739,7 @@ HttpProxyMiddleware
This middleware sets the HTTP proxy to use for requests, by setting the
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
Like the Python standard library module :mod:`urllib.request`, it obeys
the following environment variables:
* ``http_proxy``
@ -751,9 +751,6 @@ HttpProxyMiddleware
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
environment variables, and it will also ignore ``no_proxy`` environment variable.
.. _urllib: https://docs.python.org/2/library/urllib.html
.. _urllib2: https://docs.python.org/2/library/urllib2.html
RedirectMiddleware
------------------
@ -829,6 +826,7 @@ REDIRECT_MAX_TIMES
Default: ``20``
The maximum number of redirections that will be followed for a single request.
After this maximum, the request's response is returned as is.
MetaRefreshMiddleware
---------------------
@ -1036,8 +1034,7 @@ Scrapy uses this parser by default.
RobotFileParser
~~~~~~~~~~~~~~~
Based on `RobotFileParser
<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
Based on :class:`~urllib.robotparser.RobotFileParser`:
* is Python's built-in robots.txt_ parser

17
docs/topics/dynamic-content.rst
View File

@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is
probably not your request, but the target server. The target server might be
buggy, overloaded, or :ref:`banning <bans>` some of your requests.
Note that to translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
.. _topics-handling-response-formats:
Handling different response formats
@ -115,7 +118,7 @@ data from it depends on the type of response:
- If the response is HTML or XML, use :ref:`selectors
<topics-selectors>` as usual.
- If the response is JSON, use `json.loads`_ to load the desired data from
- If the response is JSON, use :func:`json.loads` to load the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`::
data = json.loads(response.text)
@ -130,8 +133,9 @@ data from it depends on the type of response:
- If the response is JavaScript, or HTML with a ``<script/>`` element
containing the desired data, see :ref:`topics-parsing-javascript`.
- If the response is CSS, use a `regular expression`_ to extract the desired
data from :attr:`response.text <scrapy.http.TextResponse.text>`.
- If the response is CSS, use a :doc:`regular expression <library/re>` to
extract the desired data from
:attr:`response.text <scrapy.http.TextResponse.text>`.
.. _topics-parsing-images:
@ -168,8 +172,9 @@ JavaScript code:
Once you have a string with the JavaScript code, you can extract the desired
data from it:
- You might be able to use a `regular expression`_ to extract the desired
data in JSON format, which you can then parse with `json.loads`_.
- You might be able to use a :doc:`regular expression <library/re>` to
extract the desired data in JSON format, which you can then parse with
:func:`json.loads`.
For example, if the JavaScript code contains a separate line like
``var data = {"field": "value"};`` you can extract that data as follows:
@ -241,9 +246,7 @@ along with `scrapy-selenium`_ for seamless integration.
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _js2xml: https://github.com/scrapinghub/js2xml
.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
.. _pytesseract: https://github.com/madmaze/pytesseract
.. _regular expression: https://docs.python.org/3/library/re.html
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
.. _Selenium: https://www.selenium.dev/

4
docs/topics/email.rst
View File

@ -7,7 +7,7 @@ Sending e-mail
.. module:: scrapy.mail
:synopsis: Email sending facility
Although Python makes sending e-mails relatively easy via the `smtplib`_
Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
library, Scrapy provides its own facility for sending e-mails which is very
easy to use and it's implemented using :doc:`Twisted non-blocking IO
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
@ -15,8 +15,6 @@ IO of the crawler. It also provides a simple API for sending attachments and
it's very easy to configure, with a few :ref:`settings
<topics-email-settings>`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
Quick example
=============

20
docs/topics/exporters.rst
View File

@ -311,7 +311,7 @@ CsvItemExporter
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
:func:`csv.writer` function, so you can use any :func:`csv.writer` function
argument to customize this exporter.
A typical output of this exporter would be::
@ -320,8 +320,6 @@ CsvItemExporter
Color TV,1200
DVD player,200
.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
PickleItemExporter
------------------
@ -335,15 +333,13 @@ PickleItemExporter
:param protocol: The pickle protocol to use.
:type protocol: int
For more information, refer to the `pickle module documentation`_.
For more information, see :mod:`pickle`.
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
PprintItemExporter
------------------
@ -372,8 +368,8 @@ JsonItemExporter
Exports Items in JSON format to the specified file-like object, writing all
objects as a list of objects. The additional ``__init__`` method arguments are
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -393,8 +389,6 @@ JsonItemExporter
stream-friendly format, consider using :class:`JsonLinesItemExporter`
instead, or splitting the output in multiple chunks.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
JsonLinesItemExporter
---------------------
@ -403,8 +397,8 @@ JsonLinesItemExporter
Exports Items in JSON format to the specified file-like object, writing one
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
``__init__`` method argument to customize this exporter.
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@ -417,8 +411,6 @@ JsonLinesItemExporter
Unlike the one produced by :class:`JsonItemExporter`, the format produced by
this exporter is well suited for serializing large amounts of data.
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
MarshalItemExporter
-------------------

3
docs/topics/extensions.rst
View File

@ -364,7 +364,7 @@ Debugger extension
.. class:: Debugger
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process when a `SIGUSR2`_
signal is received. After the debugger is exited, the Scrapy process continues
running normally.
@ -372,5 +372,4 @@ For more info see `Debugging in Python`_.
This extension only works on POSIX-compliant platforms (i.e. not Windows).
.. _Python debugger: https://docs.python.org/2/library/pdb.html
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/

33
docs/topics/items.rst
View File

@ -15,8 +15,8 @@ especially in a larger project with many spiders.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
their available fields.
They provide an API similar to :class:`dict` API with a convenient syntax
for declaring their available fields.
Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
@ -24,8 +24,6 @@ serialization can be customized using Item fields metadata, :mod:`trackref`
tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
.. _topics-items-declaring:
Declaring Items
@ -79,7 +77,7 @@ Working with Items
Here are some examples of common tasks performed with items, using the
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
notice the API is very similar to the `dict API`_.
notice the API is very similar to the :class:`dict` API.
Creating items
--------------
@ -145,7 +143,7 @@ KeyError: 'Product does not support field: lala'
Accessing all populated values
------------------------------
To access all populated values, just use the typical `dict API`_:
To access all populated values, just use the typical :class:`dict` API:
>>> product.keys()
['price', 'name']
@ -162,11 +160,9 @@ Copying items
To copy an item, you must first decide whether you want a shallow copy or a
deep copy.
If your item contains mutable_ values like lists or dictionaries, a shallow
copy will keep references to the same mutable values across all different
copies.
.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
If your item contains :term:`mutable` values like lists or dictionaries,
a shallow copy will keep references to the same mutable values across all
different copies.
For example, if you have an item with a list of tags, and you create a shallow
copy of that item, both the original item and the copy have the same list of
@ -175,9 +171,7 @@ other item as well.
If that is not the desired behavior, use a deep copy instead.
See the `documentation of the copy module`_ for more information.
.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
See :mod:`copy` for more information.
To create a shallow copy of an item, you can either call
:meth:`~scrapy.item.Item.copy` on an existing item
@ -235,8 +229,8 @@ Item objects
Return a new Item optionally initialized from the given argument.
Items replicate the standard `dict API`_, including its ``__init__`` method, and
also provide the following additional API members:
Items replicate the standard :class:`dict` API, including its ``__init__``
method, and also provide the following additional API members:
.. automethod:: copy
@ -249,22 +243,17 @@ Item objects
:class:`Field` objects used in the :ref:`Item declaration
<topics-items-declaring>`.
.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
Field objects
=============
.. class:: Field([arg])
The :class:`Field` class is just an alias to the built-in `dict`_ class and
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
doesn't provide any extra functionality or attributes. In other words,
:class:`Field` objects are plain-old Python dicts. A separate class is used
to support the :ref:`item declaration syntax <topics-items-declaring>`
based on class attributes.
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
Other classes related to Item
=============================

27
docs/topics/logging.rst
View File

@ -9,8 +9,7 @@ Logging
explicit calls to the Python standard logging. Keep reading to learn more
about the new logging system.
Scrapy uses `Python's builtin logging system
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
Scrapy uses :mod:`logging` for event logging. We'll
provide some simple examples to get you started, but for more advanced
use-cases it's strongly suggested to read thoroughly its documentation.
@ -83,10 +82,10 @@ path::
.. seealso::
Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
Module logging, :doc:`HowTo <howto/logging>`
Basic Logging Tutorial
Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
Module logging, :ref:`Loggers <logger>`
Further documentation on loggers
.. _topics-logging-from-spiders:
@ -165,14 +164,12 @@ possible levels listed in :ref:`topics-logging-levels`.
:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
used as layouts for all messages. Those strings can contain any placeholders
listed in `logging's logrecord attributes docs
<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
`datetime's strftime and strptime directives
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
respectively.
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
component that prints the log. It is unset by default, hence logs contain the
component that prints the log. It is unset by default, hence logs contain the
Scrapy component responsible for that log output.
Command-line options
@ -190,7 +187,7 @@ to override some of the Scrapy settings regarding logging.
.. seealso::
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
Module :mod:`logging.handlers`
Further documentation on available handlers
.. _custom-log-formats:
@ -201,7 +198,7 @@ Custom Log Formats
A custom log format can be set for different actions by extending
:class:`~scrapy.logformatter.LogFormatter` class and making
:setting:`LOG_FORMATTER` point to your new class.
.. autoclass:: scrapy.logformatter.LogFormatter
:members:
@ -256,10 +253,10 @@ scrapy.utils.log module
In that case, its usage is not required but it's recommended.
Another option when running custom scripts is to manually configure the logging.
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
To do this you can use :func:`logging.basicConfig` to set a basic root handler.
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
so it is recommended to only use `logging.basicConfig()`_ together with
so it is recommended to only use :func:`logging.basicConfig` together with
:class:`~scrapy.crawler.CrawlerRunner`.
This is an example on how to redirect ``INFO`` or higher messages to a file::
@ -275,7 +272,3 @@ scrapy.utils.log module
Refer to :ref:`run-from-script` for more details about using Scrapy this
way.
.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig

5
docs/topics/practices.rst
View File

@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it.
...
process = CrawlerProcess(settings={
'FEED_FORMAT': 'json',
'FEED_URI': 'items.json'
"FEEDS": {
"items.json": {"format": "json"},
},
})
process.crawl(MySpider)

44
docs/topics/request-response.rst
View File

@ -36,7 +36,7 @@ Request objects
:type url: string
:param callback: the function that will be called with the response of this
request (once its downloaded) as its first parameter. For more information
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.spiders.Spider.parse` method will be used.
@ -174,9 +174,9 @@ Request objects
See :ref:`topics-request-meta` for a list of special meta keys
recognized by Scrapy.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.meta`` attribute.
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.meta`` attribute.
.. attribute:: Request.cb_kwargs
@ -185,11 +185,9 @@ Request objects
for new Requests, which means by default callbacks only get a :class:`Response`
object as argument.
This dict is `shallow copied`_ when the request is cloned using the
``copy()`` or ``replace()`` methods, and can also be accessed, in your
spider, from the ``response.cb_kwargs`` attribute.
.. _shallow copied: https://docs.python.org/2/library/copy.html
This dict is :doc:`shallow copied <library/copy>` when the request is
cloned using the ``copy()`` or ``replace()`` methods, and can also be
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
.. method:: Request.copy()
@ -566,12 +564,10 @@ dealing with JSON requests.
set to ``'POST'`` automatically.
:type data: JSON serializable object
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
data into JSON format.
:type dumps_kwargs: dict
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
JsonRequest usage example
-------------------------
@ -620,6 +616,12 @@ Response objects
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. attribute:: Response.url
A string containing the URL of the response.
@ -706,9 +708,19 @@ Response objects
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
Only populated for ``https`` responses, ``None`` otherwise.
.. attribute:: Response.ip_address
.. versionadded:: 2.1.0
The IP address of the server from which the Response originated.
This attribute is currently only populated by the HTTP 1.1 download
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.
@ -724,18 +736,16 @@ Response objects
Constructs an absolute url by combining the Response's :attr:`url` with
a possible relative url.
This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
making this call::
urlparse.urljoin(response.url, url)
urllib.parse.urljoin(response.url, url)
.. automethod:: Response.follow
.. automethod:: Response.follow_all
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
.. _topics-request-response-ref-response-subclasses:
Response subclasses

3
docs/topics/selectors.rst
View File

@ -14,7 +14,7 @@ achieve this, such as:
drawback: it's slow.
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
API based on `ElementTree`_. (lxml is not part of the Python standard
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
library.)
Scrapy comes with its own mechanism for extracting data. They're called
@ -36,7 +36,6 @@ defines selectors to associate those styles with specific HTML elements.
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
.. _lxml: https://lxml.de/
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
.. _XPath: https://www.w3.org/TR/xpath/all/
.. _CSS: https://www.w3.org/TR/selectors
.. _parsel: https://parsel.readthedocs.io/en/latest/

40
docs/topics/settings.rst
View File

@ -26,9 +26,7 @@ do this by using an environment variable, ``SCRAPY_SETTINGS_MODULE``.
The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
``myproject.settings``. Note that the settings module should be on the
Python `import search path`_.
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
Python :ref:`import search path <tut-searchpath>`.
.. _populating-settings:
@ -899,10 +897,9 @@ LOG_FORMAT
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
placeholders.
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
String for formatting log messages. Refer to the
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
list of available placeholders.
.. setting:: LOG_DATEFORMAT
@ -912,10 +909,9 @@ LOG_DATEFORMAT
Default: ``'%Y-%m-%d %H:%M:%S'``
String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
directives.
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
in :setting:`LOG_FORMAT`. Refer to the
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
whole list of available directives.
.. setting:: LOG_FORMATTER
@ -1116,17 +1112,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
this value if you're experiencing problems with insufficient blocking IO.
.. setting:: REDIRECT_MAX_TIMES
REDIRECT_MAX_TIMES
------------------
Default: ``20``
Defines the maximum times a request can be redirected. After this maximum the
request's response is returned as is. We used Firefox default value for the
same task.
.. setting:: REDIRECT_PRIORITY_ADJUST
REDIRECT_PRIORITY_ADJUST
@ -1422,17 +1407,6 @@ Default: ``True``
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
will be enabled (provided its extension is also enabled).
.. setting:: TELNETCONSOLE_PORT
TELNETCONSOLE_PORT
------------------
Default: ``[6023, 6073]``
The port range to use for the telnet console. If set to ``None`` or ``0``, a
dynamically assigned port is used. For more info see
:ref:`topics-telnetconsole`.
.. setting:: TEMPLATES_DIR
TEMPLATES_DIR

11
docs/topics/shell.rst
View File

@ -156,6 +156,17 @@ First, we launch the shell::
scrapy shell 'https://scrapy.org' --nolog
.. note::
Remember to always enclose URLs in quotes when running the Scrapy shell from
the command line, otherwise URLs containing arguments (i.e. the ``&`` character)
will not work.
On Windows, use double quotes instead::
scrapy shell "https://scrapy.org" --nolog
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
list of available objects and useful shortcuts (you'll notice that these lines
all start with the ``[s]`` prefix)::

134
docs/topics/signals.rst
View File

@ -16,8 +16,7 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action:
::
Here is a simple example showing how you can catch signals and perform some action::
from scrapy import signals
from scrapy import Spider
@ -52,9 +51,45 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, see the :ref:`topics-signals-ref` below to know
which ones.
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
Let's take an example::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['http://quotes.toscrape.com/page/1/']
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
# The next item will be scraped only after
# deferred (d) is fired
return d
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
.. _topics-signals-ref:
@ -66,9 +101,12 @@ Built-in signals reference
Here's the list of Scrapy built-in signals and their meaning.
engine_started
Engine signals
--------------
engine_started
~~~~~~~~~~~~~~
.. signal:: engine_started
.. function:: engine_started()
@ -81,7 +119,7 @@ engine_started
getting fired before :signal:`spider_opened`.
engine_stopped
--------------
~~~~~~~~~~~~~~
.. signal:: engine_stopped
.. function:: engine_stopped()
@ -91,9 +129,20 @@ engine_stopped
This signal supports returning deferreds from its handlers.
item_scraped
Item signals
------------
.. note::
As at max :setting:`CONCURRENT_ITEMS` items are processed in
parallel, many deferreds are fired together using
:class:`~twisted.internet.defer.DeferredList`. Hence the next
batch waits for the :class:`~twisted.internet.defer.DeferredList`
to fire and then runs the respective item signal handler for
the next batch of scraped items.
item_scraped
~~~~~~~~~~~~
.. signal:: item_scraped
.. function:: item_scraped(item, response, spider)
@ -112,7 +161,7 @@ item_scraped
:type response: :class:`~scrapy.http.Response` object
item_dropped
------------
~~~~~~~~~~~~
.. signal:: item_dropped
.. function:: item_dropped(item, response, exception, spider)
@ -137,7 +186,7 @@ item_dropped
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
item_error
------------
~~~~~~~~~~
.. signal:: item_error
.. function:: item_error(item, response, spider, failure)
@ -159,29 +208,11 @@ item_error
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
bytes_received
Spider signals
--------------
.. signal:: bytes_received
.. function:: bytes_received(data, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
received for a specific request. This signal might be fired multiple
times for the same request, with partial data each time.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type spider: :class:`bytes` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
spider_closed
-------------
~~~~~~~~~~~~~
.. signal:: spider_closed
.. function:: spider_closed(spider, reason)
@ -204,7 +235,7 @@ spider_closed
:type reason: str
spider_opened
-------------
~~~~~~~~~~~~~
.. signal:: spider_opened
.. function:: spider_opened(spider)
@ -219,7 +250,7 @@ spider_opened
:type spider: :class:`~scrapy.spiders.Spider` object
spider_idle
-----------
~~~~~~~~~~~
.. signal:: spider_idle
.. function:: spider_idle(spider)
@ -249,7 +280,7 @@ spider_idle
due to duplication).
spider_error
------------
~~~~~~~~~~~~
.. signal:: spider_error
.. function:: spider_error(failure, response, spider)
@ -267,8 +298,11 @@ spider_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
Request signals
---------------
request_scheduled
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: request_scheduled
.. function:: request_scheduled(request, spider)
@ -285,7 +319,7 @@ request_scheduled
:type spider: :class:`~scrapy.spiders.Spider` object
request_dropped
---------------
~~~~~~~~~~~~~~~
.. signal:: request_dropped
.. function:: request_dropped(request, spider)
@ -302,7 +336,7 @@ request_dropped
:type spider: :class:`~scrapy.spiders.Spider` object
request_reached_downloader
---------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_reached_downloader
.. function:: request_reached_downloader(request, spider)
@ -318,7 +352,7 @@ request_reached_downloader
:type spider: :class:`~scrapy.spiders.Spider` object
request_left_downloader
-----------------------
~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_left_downloader
.. function:: request_left_downloader(request, spider)
@ -336,8 +370,32 @@ request_left_downloader
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
bytes_received
~~~~~~~~~~~~~~
.. signal:: bytes_received
.. function:: bytes_received(data, request, spider)
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
received for a specific request. This signal might be fired multiple
times for the same request, with partial data each time.
This signal does not support returning deferreds from its handlers.
:param data: the data received by the download handler
:type spider: :class:`bytes` object
:param request: the request that generated the response
:type request: :class:`~scrapy.http.Request` object
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.spiders.Spider` object
Response signals
----------------
response_received
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: response_received
.. function:: response_received(response, request, spider)
@ -357,7 +415,7 @@ response_received
:type spider: :class:`~scrapy.spiders.Spider` object
response_downloaded
-------------------
~~~~~~~~~~~~~~~~~~~
.. signal:: response_downloaded
.. function:: response_downloaded(response, request, spider)

10
docs/topics/spider-middleware.rst
View File

@ -140,7 +140,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type response: :class:`~scrapy.http.Response` object
:param exception: the exception raised
:type exception: `Exception`_ object
:type exception: :exc:`Exception` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
@ -173,20 +173,16 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
:type spider: :class:`~scrapy.spiders.Spider` object
.. method:: from_crawler(cls, crawler)
If present, this classmethod is called to create a middleware instance
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
of the middleware. Crawler object provides access to all Scrapy core
components like settings and signals; it is a way for middleware to
access them and hook its functionality into Scrapy.
:param crawler: crawler that uses this middleware
:type crawler: :class:`~scrapy.crawler.Crawler` object
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
.. _topics-spider-middleware-ref:
Built-in spider middleware reference

4
docs/topics/spiders.rst
View File

@ -298,9 +298,7 @@ Keep in mind that spider arguments are only strings.
The spider will not do any parsing on its own.
If you were to set the ``start_urls`` attribute from the command line,
you would have to parse it on your own into a list
using something like
`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
using something like :func:`ast.literal_eval` or :func:`json.loads`
and then set it as an attribute.
Otherwise, you would cause iteration over a ``start_urls`` string
(a very common python pitfall)

11
docs/topics/telnetconsole.rst
View File

@ -40,10 +40,10 @@ the console you need to type::
Connected to localhost.
Escape character is '^]'.
Username:
Password:
Password:
>>>
By default Username is ``scrapy`` and Password is autogenerated. The
By default Username is ``scrapy`` and Password is autogenerated. The
autogenerated Password can be seen on Scrapy logs like the example below::
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
@ -63,7 +63,7 @@ Available variables in the telnet console
=========================================
The telnet console is like a regular Python shell running inside the Scrapy
process, so you can do anything from it including importing new modules, etc.
process, so you can do anything from it including importing new modules, etc.
However, the telnet console comes with some default variables defined for
convenience:
@ -89,13 +89,11 @@ convenience:
+----------------+-------------------------------------------------------------------+
| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
| ``p`` | a shortcut to the `pprint.pprint`_ function |
| ``p`` | a shortcut to the :func:`pprint.pprint` function |
+----------------+-------------------------------------------------------------------+
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
+----------------+-------------------------------------------------------------------+
.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
Telnet console usage examples
=============================
@ -208,4 +206,3 @@ Default: ``None``
The password used for the telnet console, default behaviour is to have it
autogenerated

62
pytest.ini
View File

@ -35,27 +35,27 @@ flake8-ignore =
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731
scrapy/commands/fetch.py E401 E501 E128
scrapy/commands/genspider.py E128 E501
scrapy/commands/parse.py E128 E501
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
scrapy/commands/shell.py E128 E501
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127 E502
scrapy/core/engine.py E501 E128 E127
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128 W504
scrapy/core/spidermw.py E501 E731 E126
scrapy/core/spidermw.py E501 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E241
scrapy/core/downloader/webclient.py E731 E501 E128 E126
scrapy/core/downloader/middleware.py E501
scrapy/core/downloader/tls.py E501
scrapy/core/downloader/webclient.py E501 E128 E126
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
scrapy/core/downloader/handlers/http10.py E501
@ -90,16 +90,16 @@ flake8-ignore =
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501 E731
scrapy/linkextractors/__init__.py E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501
# scrapy/loader
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/__init__.py E501
scrapy/pipelines/files.py E116 E501 E266
scrapy/pipelines/images.py E265 E501
scrapy/pipelines/media.py E125 E501 E266
scrapy/pipelines/files.py E116 E501
scrapy/pipelines/images.py E501
scrapy/pipelines/media.py E125 E501
# scrapy/selector
scrapy/selector/__init__.py F403
scrapy/selector/unified.py E501 E111
@ -124,7 +124,7 @@ flake8-ignore =
scrapy/utils/datatypes.py E501
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
scrapy/utils/deprecate.py E128 E501 E127
scrapy/utils/gz.py E501 W504
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
@ -149,14 +149,14 @@ flake8-ignore =
scrapy/__init__.py E402 E501
scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/dupefilters.py E501
scrapy/exceptions.py E501
scrapy/exporters.py E501
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
scrapy/mail.py E402 E128 E501
scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/resolver.py E501
@ -178,13 +178,13 @@ flake8-ignore =
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawl.py E501 E741
tests/test_crawler.py F841 E501
tests/test_dependencies.py F841 E501
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
@ -197,24 +197,24 @@ flake8-ignore =
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E128 E124
tests/test_exporters.py E501 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
tests/test_http_response.py E501 E128 E265
tests/test_http_response.py E501 E128
tests/test_item.py E128 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E501 E731 E741 E128 E117 E241
tests/test_loader.py E501 E741 E128 E117
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128 E126
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E731 E128 E502
tests/test_pipeline_media.py E501 E741 E128
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
@ -226,7 +226,7 @@ flake8-ignore =
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
tests/test_squeues.py E501 E741
tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E128
@ -235,16 +235,16 @@ flake8-ignore =
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E241
tests/test_utils_iterators.py E501 E128 E129
tests/test_utils_log.py E741
tests/test_utils_python.py E501 E731
tests/test_utils_python.py E501
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731
tests/test_utils_signal.py E741 F841
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E123 E126
tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501

2
scrapy/VERSION
View File

@ -1 +1 @@
2.0.0
2.1.0

3
scrapy/commands/__init__.py
View File

@ -23,7 +23,8 @@ class ScrapyCommand:
self.settings = None # set in scrapy.cmdline
def set_crawler(self, crawler):
assert not hasattr(self, '_crawler'), "crawler already set"
if hasattr(self, '_crawler'):
raise RuntimeError("crawler already set")
self._crawler = crawler
def syntax(self):

4
scrapy/commands/fetch.py
View File

@ -49,8 +49,8 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
cb = lambda x: self._print_response(x, opts)
request = Request(args[0], callback=cb, dont_filter=True)
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:

7
scrapy/commands/genspider.py
View File

@ -90,8 +90,7 @@ class Command(ScrapyCommand):
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() \
for s in module.split('_'))
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -102,8 +101,8 @@ class Command(ScrapyCommand):
spider_file = "%s.py" % join(spiders_dir, module)
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r " % (name, \
template_name), end=('' if spiders_module else '\n'))
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))

5
scrapy/commands/parse.py
View File

@ -146,9 +146,8 @@ class Command(ScrapyCommand):
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):

2
scrapy/commands/shell.py
View File

@ -37,7 +37,7 @@ class Command(ScrapyCommand):
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):

6
scrapy/contracts/default.py
View File

@ -58,7 +58,11 @@ class ReturnsContract(Contract):
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
assert len(self.args) in [1, 2, 3]
if len(self.args) not in [1, 2, 3]:
raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
% len(self.args)
)
self.obj_name = self.args[0] or None
self.obj_type = self.objects[self.obj_name]

2
scrapy/core/downloader/__init__.py
View File

@ -173,7 +173,7 @@ class Downloader:
return response
dfd.addCallback(_downloaded)
# 3. After response arrives, remove the request from transferring
# 3. After response arrives, remove the request from transferring
# state to free up the transferring slot so it can be used by the
# following requests (perhaps those which came from the downloader
# middleware itself)

68
scrapy/core/downloader/handlers/http11.py
View File

@ -1,5 +1,6 @@
"""Download handlers for http and https schemes"""
import ipaddress
import logging
import re
import warnings
@ -346,20 +347,6 @@ class ScrapyAgent:
headers.removeHeader(b'Proxy-Authorization')
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
elif method == b'POST':
# Setting Content-Length: 0 even for POST requests is not a
# MUST per HTTP RFCs, but it's common behavior, and some
# servers require this, otherwise returning HTTP 411 Length required
#
# RFC 7230#section-3.3.2:
# "a Content-Length header field is normally sent in a POST
# request even when the value is 0 (indicating an empty payload body)."
#
# Twisted < 17 will not add "Content-Length: 0" by itself;
# Twisted >= 17 fixes this;
# Using a producer with an empty-string sends `0` as Content-Length
# for all versions of Twisted.
bodyproducer = _RequestBodyProducer(b'')
else:
bodyproducer = None
start_time = time()
@ -392,7 +379,13 @@ class ScrapyAgent:
def _cb_bodyready(self, txresponse, request):
# deliverBody hangs for responses without body
if txresponse.length == 0:
return txresponse, b'', None, None
return {
"txresponse": txresponse,
"body": b"",
"flags": None,
"certificate": None,
"ip_address": None,
}
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
@ -436,12 +429,17 @@ class ScrapyAgent:
return d
def _cb_bodydone(self, result, request, url):
txresponse, body, flags, certificate = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
return respcls(url=url, status=status, headers=headers, body=body,
flags=flags, certificate=certificate)
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
return respcls(
url=url,
status=int(result["txresponse"].code),
headers=headers,
body=result["body"],
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
)
@implementer(IBodyProducer)
@ -476,6 +474,7 @@ class _ResponseReader(protocol.Protocol):
self._reached_warnsize = False
self._bytes_received = 0
self._certificate = None
self._ip_address = None
self._crawler = crawler
def connectionMade(self):
@ -483,6 +482,9 @@ class _ResponseReader(protocol.Protocol):
with suppress(AttributeError):
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
if self._ip_address is None:
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered data.
if self._finished.called:
@ -521,16 +523,34 @@ class _ResponseReader(protocol.Protocol):
body = self._bodybuf.getvalue()
if reason.check(ResponseDone):
self._finished.callback((self._txresponse, body, None, self._certificate))
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": None,
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
if reason.check(PotentialDataLoss):
self._finished.callback((self._txresponse, body, ['partial'], self._certificate))
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["partial"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if not self._fail_on_dataloss:
self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate))
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["dataloss"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
elif not self._fail_on_dataloss_warned:

41
scrapy/core/downloader/middleware.py
View File

@ -35,38 +35,45 @@ class DownloaderMiddlewareManager(MiddlewareManager):
for method in self.methods['process_request']:
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, response.__class__.__name__))
raise _InvalidOutput(
"Middleware %s.process_request must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, response.__class__.__name__)
)
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
return response
return (yield download_func(request=request, spider=spider))
@defer.inlineCallbacks
def process_response(response):
assert response is not None, 'Received None in process_response'
if isinstance(response, Request):
defer.returnValue(response)
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
return response
for method in self.methods['process_response']:
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
raise _InvalidOutput(
"Middleware %s.process_response must return Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
)
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
return response
return response
@defer.inlineCallbacks
def process_exception(_failure):
exception = _failure.value
def process_exception(failure):
exception = failure.value
for method in self.methods['process_exception']:
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
raise _InvalidOutput(
"Middleware %s.process_exception must return None, Response or Request, got %s"
% (method.__self__.__class__.__name__, type(response))
)
if response:
defer.returnValue(response)
defer.returnValue(_failure)
return response
return failure
deferred = mustbe_deferred(process_request, request)
deferred.addErrback(process_exception)

4
scrapy/core/downloader/tls.py
View File

@ -20,8 +20,8 @@ METHOD_TLSv12 = 'TLSv1.2'
openssl_methods = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only

9
scrapy/core/downloader/webclient.py
View File

@ -14,13 +14,12 @@ from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
# Assume parsed is urlparse-d from Request.url,
# which was passed via safe_url_string and is ascii-only.
b = lambda s: to_bytes(s, encoding='ascii')
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
path = b(path)
host = b(parsed.hostname)
path = to_bytes(path, encoding="ascii")
host = to_bytes(parsed.hostname, encoding="ascii")
port = parsed.port
scheme = b(parsed.scheme)
netloc = b(parsed.netloc)
scheme = to_bytes(parsed.scheme, encoding="ascii")
netloc = to_bytes(parsed.netloc, encoding="ascii")
if port is None:
port = 443 if scheme == b'https' else 80
return scheme, netloc, host, port, path

31
scrapy/core/engine.py
View File

@ -73,7 +73,8 @@ class ExecutionEngine:
@defer.inlineCallbacks
def start(self):
"""Start the execution engine"""
assert not self.running, "Engine already running"
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
@ -82,7 +83,8 @@ class ExecutionEngine:
def stop(self):
"""Stop the execution engine gracefully"""
assert self.running, "Engine not running"
if not self.running:
raise RuntimeError("Engine not running")
self.running = False
dfd = self._close_all_spiders()
return dfd.addBoth(lambda _: self._finish_stopping_engine())
@ -165,7 +167,11 @@ class ExecutionEngine:
return d
def _handle_downloader_output(self, response, request, spider):
assert isinstance(response, (Request, Response, Failure)), response
if not isinstance(response, (Request, Response, Failure)):
raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r"
% (type(response), response)
)
# downloader middleware can return requests (for example, redirects)
if isinstance(response, Request):
self.crawl(response, spider)
@ -205,8 +211,8 @@ class ExecutionEngine:
return not bool(self.slot)
def crawl(self, request, spider):
assert spider in self.open_spiders, \
"Spider %r not opened when crawling: %s" % (spider.name, request)
if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
self.schedule(request, spider)
self.slot.nextcall.schedule()
@ -232,7 +238,11 @@ class ExecutionEngine:
slot.add_request(request)
def _on_success(response):
assert isinstance(response, (Response, Request))
if not isinstance(response, (Response, Request)):
raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r"
% (type(response), response)
)
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
@ -253,8 +263,8 @@ class ExecutionEngine:
@defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True):
assert self.has_capacity(), "No free spider slot when opening %r" % \
spider.name
if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name)
logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler)
@ -277,10 +287,9 @@ class ExecutionEngine:
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = self.signals.send_catch_log(signal=signals.spider_idle, \
res = self.signals.send_catch_log(signal=signals.spider_idle,
spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
for _, x in res):
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
return
if self.spider_is_idle(spider):

6
scrapy/core/scraper.py
View File

@ -123,7 +123,11 @@ class Scraper:
def _scrape(self, response, request, spider):
"""Handle the downloaded response or failure through the spider
callback/errback"""
assert isinstance(response, (Response, Failure))
if not isinstance(response, (Response, Failure)):
raise TypeError(
"Incorrect type: expected Response or Failure, got %s: %r"
% (type(response), response)
)
dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)

3
scrapy/crawler.py
View File

@ -78,7 +78,8 @@ class Crawler:
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
assert not self.crawling, "Crawling already taking place"
if self.crawling:
raise RuntimeError("Crawling already taking place")
self.crawling = True
try:

2
scrapy/dupefilters.py
View File

@ -61,7 +61,7 @@ class RFPDupeFilter(BaseDupeFilter):
def log(self, request, spider):
if self.debug:
msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
args = {'request': request, 'referer': referer_str(request) }
args = {'request': request, 'referer': referer_str(request)}
self.logger.debug(msg, args, extra={'spider': spider})
elif self.logdupes:
msg = ("Filtered duplicate request: %(request)s"

6
scrapy/http/request/__init__.py
View File

@ -24,7 +24,8 @@ class Request(object_ref):
self.method = str(method).upper()
self._set_url(url)
self._set_body(body)
assert isinstance(priority, int), "Request priority not an integer: %r" % priority
if not isinstance(priority, int):
raise TypeError("Request priority not an integer: %r" % priority)
self.priority = priority
if callback is not None and not callable(callback):
@ -129,6 +130,9 @@ class Request(object_ref):
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
may modify the :class:`~scrapy.http.Request` object.
To translate a cURL command into a Scrapy request,
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
"""
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
request_kwargs.update(kwargs)

7
scrapy/http/response/__init__.py
View File

@ -17,7 +17,8 @@ from scrapy.utils.trackref import object_ref
class Response(object_ref):
def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None):
def __init__(self, url, status=200, headers=None, body=b'', flags=None,
request=None, certificate=None, ip_address=None):
self.headers = Headers(headers or {})
self.status = int(status)
self._set_body(body)
@ -25,6 +26,7 @@ class Response(object_ref):
self.request = request
self.flags = [] if flags is None else list(flags)
self.certificate = certificate
self.ip_address = ip_address
@property
def cb_kwargs(self):
@ -87,7 +89,8 @@ class Response(object_ref):
"""Create a new Response with the same attributes except for those
given new values.
"""
for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']:
for x in ['url', 'status', 'headers', 'body',
'request', 'flags', 'certificate', 'ip_address']:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)

4
scrapy/item.py
View File

@ -121,9 +121,7 @@ class DictItem(MutableMapping, BaseItem):
return self.__class__(self)
def deepcopy(self):
"""Return a `deep copy`_ of this item.
.. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
"""Return a :func:`~copy.deepcopy` of this item.
"""
return deepcopy(self)

10
scrapy/linkextractors/__init__.py
View File

@ -45,8 +45,14 @@ IGNORED_EXTENSIONS = [
_re_type = type(re.compile("", 0))
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
def _matches(url, regexs):
return any(r.search(url) for r in regexs)
def _is_valid_url(url):
return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor:

6
scrapy/linkextractors/lxmlhtml.py
View File

@ -98,11 +98,9 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
strip=True, restrict_text=None):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
lx = LxmlParserLinkExtractor(
tag=tag_func,
attr=attr_func,
tag=lambda x: x in tags,
attr=lambda x: x in attrs,
unique=unique,
process=process_value,
strip=strip,

4
scrapy/mail.py
View File

@ -115,8 +115,8 @@ class MailSender:
from twisted.mail.smtp import ESMTPSenderFactory
msg = BytesIO(msg)
d = defer.Deferred()
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
requireTransportSecurity=self.smtptls)
factory.noisy = False

8
scrapy/pipelines/files.py
View File

@ -106,7 +106,8 @@ class S3FilesStore:
else:
from boto.s3.connection import S3Connection
self.S3Connection = S3Connection
assert uri.startswith('s3://')
if not uri.startswith("s3://"):
raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri)
self.bucket, self.prefix = uri[5:].split('/', 1)
def stat_file(self, path, info):
@ -266,7 +267,8 @@ class FTPFilesStore:
USE_ACTIVE_MODE = None
def __init__(self, uri):
assert uri.startswith('ftp://')
if not uri.startswith("ftp://"):
raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri)
u = urlparse(uri)
self.port = u.port
self.host = u.hostname
@ -500,7 +502,7 @@ class FilesPipeline(MediaPipeline):
spider.crawler.stats.inc_value('file_count', spider=spider)
spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)
### Overridable Interface
# Overridable Interface
def get_media_requests(self, item, info):
return [Request(x) for x in item.get(self.files_urls_field, [])]

2
scrapy/pipelines/images.py
View File

@ -14,7 +14,7 @@ from scrapy.utils.python import to_bytes
from scrapy.http import Request
from scrapy.settings import Settings
from scrapy.exceptions import DropItem
#TODO: from scrapy.pipelines.media import MediaPipeline
# TODO: from scrapy.pipelines.media import MediaPipeline
from scrapy.pipelines.files import FileException, FilesPipeline

26
scrapy/pipelines/media.py
View File

@ -1,7 +1,7 @@
import functools
import logging
from collections import defaultdict
from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
from twisted.internet.defer import Deferred, DeferredList
from twisted.python.failure import Failure
from scrapy.settings import Settings
@ -141,24 +141,26 @@ class MediaPipeline:
# This code fixes a memory leak by avoiding to keep references to
# the Request and Response objects on the Media Pipeline cache.
#
# Twisted inline callbacks pass return values using the function
# twisted.internet.defer.returnValue, which encapsulates the return
# value inside a _DefGen_Return base exception.
#
# What happens when the media_downloaded callback raises another
# What happens when the media_downloaded callback raises an
# exception, for example a FileException('download-error') when
# the Response status code is not 200 OK, is that it stores the
# _DefGen_Return exception on the FileException context.
# the Response status code is not 200 OK, is that the original
# StopIteration exception (which in turn contains the failed
# Response and by extension, the original Request) gets encapsulated
# within the FileException context.
#
# Originally, Scrapy was using twisted.internet.defer.returnValue
# inside functions decorated with twisted.internet.defer.inlineCallbacks,
# encapsulating the returned Response in a _DefGen_Return exception
# instead of a StopIteration.
#
# To avoid keeping references to the Response and therefore Request
# objects on the Media Pipeline cache, we should wipe the context of
# the exception encapsulated by the Twisted Failure when its a
# _DefGen_Return instance.
# the encapsulated exception when it is a StopIteration instance
#
# This problem does not occur in Python 2.7 since we don't have
# Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
context = getattr(result.value, '__context__', None)
if isinstance(context, _DefGen_Return):
if isinstance(context, StopIteration):
setattr(result.value, '__context__', None)
info.downloading.remove(fp)
@ -166,7 +168,7 @@ class MediaPipeline:
for wad in info.waiting.pop(fp):
defer_result(result).chainDeferred(wad)
### Overridable Interface
# Overridable Interface
def media_to_download(self, request, info):
"""Check request before starting download"""
pass

2
scrapy/responsetypes.py
View File

@ -71,7 +71,7 @@ class ResponseTypes:
cls = Response
if b'Content-Type' in headers:
cls = self.from_content_type(
content_type=headers[b'Content-type'],
content_type=headers[b'Content-Type'],
content_encoding=headers.get(b'Content-Encoding')
)
if cls is Response and b'Content-Disposition' in headers:

15
scrapy/utils/deprecate.py
View File

@ -7,9 +7,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning
def attribute(obj, oldattr, newattr, version='0.12'):
cname = obj.__class__.__name__
warnings.warn("%s.%s attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead" % \
(cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)
warnings.warn(
"%s.%s attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead"
% (cname, oldattr, version, cname, newattr),
ScrapyDeprecationWarning,
stacklevel=3)
def create_deprecated_class(name, new_class, clsdict=None,
@ -17,10 +20,10 @@ def create_deprecated_class(name, new_class, clsdict=None,
warn_once=True,
old_class_path=None,
new_class_path=None,
subclass_warn_message="{cls} inherits from "\
"deprecated class {old}, please inherit "\
subclass_warn_message="{cls} inherits from "
"deprecated class {old}, please inherit "
"from {new}.",
instance_warn_message="{cls} is deprecated, "\
instance_warn_message="{cls} is deprecated, "
"instantiate {new} instead."):
"""
Return a "deprecated" class that causes its subclasses to issue a warning.

10
scrapy/utils/iterators.py
View File

@ -128,10 +128,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
type(obj).__name__)
if not isinstance(obj, expected_types):
expected_types_str = " or ".join(t.__name__ for t in expected_types)
raise TypeError(
"Object %r must be %s, not %s"
% (obj, expected_types_str, type(obj).__name__)
)
if isinstance(obj, Response):
if not unicode:
return obj.body

3
scrapy/utils/reactor.py
View File

@ -9,7 +9,8 @@ from scrapy.utils.misc import load_object
def listen_tcp(portrange, host, factory):
"""Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
if len(portrange) > 2:
raise ValueError("invalid portrange: %s" % portrange)
if not portrange:
return reactor.listenTCP(0, factory, interface=host)
if not hasattr(portrange, '__iter__'):

31
scrapy/utils/reqser.py
View File

@ -1,6 +1,8 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
import inspect
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@ -68,20 +70,6 @@ def request_from_dict(d, spider=None):
)
def _is_private_method(name):
return name.startswith('__') and not name.endswith('__')
def _mangle_private_name(obj, func, name):
qualname = getattr(func, '__qualname__', None)
if qualname is None:
classname = obj.__class__.__name__.lstrip('_')
return '_%s%s' % (classname, name)
else:
splits = qualname.split('.')
return '_%s%s' % (splits[-2], splits[-1])
def _find_method(obj, func):
if obj:
try:
@ -90,10 +78,17 @@ def _find_method(obj, func):
pass
else:
if func_self is obj:
name = func.__func__.__name__
if _is_private_method(name):
return _mangle_private_name(obj, func, name)
return name
members = inspect.getmembers(obj, predicate=inspect.ismethod)
for name, obj_func in members:
# We need to use __func__ to access the original
# function object because instance method objects
# are generated each time attribute is retrieved from
# instance.
#
# Reference: The standard type hierarchy
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
raise ValueError("Function %s is not a method of: %s" % (func, obj))

38
tests/CrawlerRunner/ip_address.py Normal file
View File

@ -0,0 +1,38 @@
from urllib.parse import urlparse
from twisted.internet import reactor
from twisted.names.client import createResolver
from scrapy import Spider, Request
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from tests.mockserver import MockServer, MockDNSServer
class LocalhostSpider(Spider):
name = "localhost_spider"
def start_requests(self):
yield Request(self.url)
def parse(self, response):
netloc = urlparse(response.url).netloc
self.logger.info("Host: %s" % netloc.split(":")[0])
self.logger.info("Type: %s" % type(response.ip_address))
self.logger.info("IP address: %s" % response.ip_address)
if __name__ == "__main__":
with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
port = urlparse(mock_http_server.http_address).port
url = "http://not.a.real.domain:{port}/echo".format(port=port)
servers = [(mock_dns_server.host, mock_dns_server.port)]
reactor.installResolver(createResolver(servers=servers))
configure_logging()
runner = CrawlerRunner()
d = runner.crawl(LocalhostSpider, url=url)
d.addBoth(lambda _: reactor.stop())
reactor.run()

99
tests/mockserver.py
View File

@ -1,3 +1,4 @@
import argparse
import json
import os
import random
@ -6,18 +7,19 @@ from subprocess import Popen, PIPE
from urllib.parse import urlencode
from OpenSSL import SSL
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import defer, reactor, ssl
from twisted.internet.task import deferLater
from twisted.names import dns, error
from twisted.names.server import DNSServerFactory
from twisted.web.resource import EncodingResourceWrapper, Resource
from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
from twisted.web.static import File
from twisted.web.test.test_webclient import PayloadResource
from twisted.web.server import GzipEncoderFactory
from twisted.web.resource import EncodingResourceWrapper
from twisted.web.util import redirectTo
from twisted.internet import reactor, ssl
from twisted.internet.task import deferLater
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
from scrapy.utils.test import get_testenv
def getarg(request, name, default=None, type=None):
@ -198,12 +200,10 @@ class Root(Resource):
return b'Scrapy mock HTTP server\n'
class MockServer():
class MockServer:
def __enter__(self):
from scrapy.utils.test import get_testenv
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'],
stdout=PIPE, env=get_testenv())
http_address = self.proc.stdout.readline().strip().decode('ascii')
https_address = self.proc.stdout.readline().strip().decode('ascii')
@ -224,11 +224,45 @@ class MockServer():
return host + path
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
def _resolve(self, name):
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
return [answer], [], []
def query(self, query, timeout=None):
if query.type == dns.A:
return defer.succeed(self._resolve(query.name.name))
return defer.fail(error.DomainError())
def lookupAllRecords(self, name, timeout=None):
return defer.succeed(self._resolve(name))
class MockDNSServer:
def __enter__(self):
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
stdout=PIPE, env=get_testenv())
host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
self.host = host
self.port = int(port)
return self
def __exit__(self, exc_type, exc_value, traceback):
self.proc.kill()
self.proc.communicate()
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
factory = ssl.DefaultOpenSSLContextFactory(
os.path.join(os.path.dirname(__file__), keyfile),
os.path.join(os.path.dirname(__file__), certfile),
)
os.path.join(os.path.dirname(__file__), keyfile),
os.path.join(os.path.dirname(__file__), certfile),
)
if cipher_string:
ctx = factory.getContext()
# disabling TLS1.2+ because it unconditionally enables some strong ciphers
@ -238,19 +272,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
if __name__ == "__main__":
root = Root()
factory = Site(root)
httpPort = reactor.listenTCP(0, factory)
contextFactory = ssl_context_factory()
httpsPort = reactor.listenSSL(0, factory, contextFactory)
parser = argparse.ArgumentParser()
parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http")
args = parser.parse_args()
def print_listening():
httpHost = httpPort.getHost()
httpsHost = httpsPort.getHost()
httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
print(httpAddress)
print(httpsAddress)
if args.type == "http":
root = Root()
factory = Site(root)
httpPort = reactor.listenTCP(0, factory)
contextFactory = ssl_context_factory()
httpsPort = reactor.listenSSL(0, factory, contextFactory)
def print_listening():
httpHost = httpPort.getHost()
httpsHost = httpsPort.getHost()
httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port)
httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port)
print(httpAddress)
print(httpsAddress)
elif args.type == "dns":
clients = [MockDNSResolver()]
factory = DNSServerFactory(clients=clients)
protocol = dns.DNSDatagramProtocol(controller=factory)
listener = reactor.listenUDP(0, protocol)
def print_listening():
host = listener.getHost()
print("%s:%s" % (host.host, host.port))
reactor.callWhenRunning(print_listening)
reactor.run()

27
tests/test_crawl.py
View File

@ -1,6 +1,9 @@
import json
import logging
import sys
from ipaddress import IPv4Address
from socket import gethostbyname
from urllib.parse import urlparse
from pytest import mark
from testfixtures import LogCapture
@ -147,9 +150,9 @@ class CrawlTestCase(TestCase):
settings = {"CONCURRENT_REQUESTS": 1}
crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
yield crawler.crawl(mockserver=self.mockserver)
#self.assertTrue(False, crawler.spider.seedsseen)
#self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
# crawler.spider.seedsseen)
self.assertTrue(
crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
crawler.spider.seedsseen)
@defer.inlineCallbacks
def test_start_requests_dupes(self):
@ -436,3 +439,21 @@ with multiples lines
self.assertIsInstance(cert, Certificate)
self.assertEqual(cert.getSubject().commonName, b"localhost")
self.assertEqual(cert.getIssuer().commonName, b"localhost")
@defer.inlineCallbacks
def test_dns_server_ip_address_none(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
self.assertIsNone(ip_address)
@defer.inlineCallbacks
def test_dns_server_ip_address(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/echo?body=test')
expected_netloc, _ = urlparse(url).netloc.split(':')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
self.assertIsInstance(ip_address, IPv4Address)
self.assertEqual(str(ip_address), gethostbyname(expected_netloc))

19
tests/test_crawler.py
View File

@ -274,9 +274,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
class CrawlerProcessSubprocess(unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
class ScriptRunnerMixin:
def run_script(self, script_name):
script_path = os.path.join(self.script_dir, script_name)
args = (sys.executable, script_path)
@ -285,6 +283,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
stdout, stderr = p.communicate()
return stderr.decode('utf-8')
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
def test_simple(self):
log = self.run_script('simple.py')
self.assertIn('Spider closed (finished)', log)
@ -332,3 +334,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
log = self.run_script("twisted_reactor_asyncio.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
def test_response_ip_address(self):
log = self.run_script("ip_address.py")
self.assertIn("INFO: Spider closed (finished)", log)
self.assertIn("INFO: Host: not.a.real.domain", log)
self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
self.assertIn("INFO: IP address: 127.0.0.1", log)

11
tests/test_downloadermiddleware_cookies.py
View File

@ -13,10 +13,9 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
class CookiesMiddlewareTest(TestCase):
def assertCookieValEqual(self, first, second, msg=None):
cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
return self.assertEqual(
sorted(cookievaleq(first)),
sorted(cookievaleq(second)), msg)
def split_cookies(cookies):
return sorted(re.split(r";\s*", cookies.decode("latin1")))
return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
def setUp(self):
self.spider = Spider('foo')
@ -202,7 +201,7 @@ class CookiesMiddlewareTest(TestCase):
assert self.mw.process_request(req4, self.spider) is None
self.assertCookieValEqual(req4.headers.get('Cookie'), b'C2=value2; galleta=dulce')
#cookies from hosts with port
# cookies from hosts with port
req5_1 = Request('http://scrapytest.org:1104/')
assert self.mw.process_request(req5_1, self.spider) is None
@ -218,7 +217,7 @@ class CookiesMiddlewareTest(TestCase):
assert self.mw.process_request(req5_3, self.spider) is None
self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1')
#skip cookie retrieval for not http request
# skip cookie retrieval for not http request
req6 = Request('file:///scrapy/sometempfile')
assert self.mw.process_request(req6, self.spider) is None
self.assertEqual(req6.headers.get('Cookie'), None)

11
tests/test_exporters.py
View File

@ -215,11 +215,12 @@ class CsvItemExporterTest(BaseItemExporterTest):
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
first = to_unicode(first)
second = to_unicode(second)
csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
for line in csv.splitlines(True)]
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
def split_csv(csv):
return [
sorted(re.split(r"(,|\s+)", line))
for line in to_unicode(csv).splitlines(True)
]
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')

6
tests/test_feedexport.py
View File

@ -433,7 +433,7 @@ class FeedExportTest(unittest.TestCase):
for file_path in FEEDS.keys():
os.remove(str(file_path))
defer.returnValue(content)
return content
@defer.inlineCallbacks
def exported_data(self, items, settings):
@ -448,7 +448,7 @@ class FeedExportTest(unittest.TestCase):
yield item
data = yield self.run_and_export(TestSpider, settings)
defer.returnValue(data)
return data
@defer.inlineCallbacks
def exported_no_data(self, settings):
@ -462,7 +462,7 @@ class FeedExportTest(unittest.TestCase):
pass
data = yield self.run_and_export(TestSpider, settings)
defer.returnValue(data)
return data
@defer.inlineCallbacks
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):

4
tests/test_http_response.py
View File

@ -438,8 +438,8 @@ class TextResponseTest(BaseResponseTest):
assert u'<span>value</span>' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
#r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
#assert u'\ufffd' in r.text, repr(r.text)
# r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
# assert u'\ufffd' in r.text, repr(r.text)
def test_selector(self):
body = b"<html><head><title>Some page</title><body></body></html>"

34
tests/test_pipeline_media.py
View File

@ -2,7 +2,7 @@ from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
from twisted.internet.defer import Deferred, inlineCallbacks
from scrapy.http import Request, Response
from scrapy.settings import Settings
@ -124,9 +124,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
# Simulate the Media Pipeline behavior to produce a Twisted Failure
try:
# Simulate a Twisted inline callback returning a Response
# The returnValue method raises an exception encapsulating the value
returnValue(response)
except BaseException as exc:
raise StopIteration(response)
except StopIteration as exc:
def_gen_return_exc = exc
try:
# Simulate the media_downloaded callback raising a FileException
@ -140,7 +139,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
# The Failure should encapsulate a FileException ...
self.assertEqual(failure.value, file_exc)
# ... and it should have the returnValue exception set as its context
# ... and it should have the StopIteration exception set as its context
self.assertEqual(failure.value.__context__, def_gen_return_exc)
# Let's calculate the request fingerprint and fake some runtime data...
@ -155,7 +154,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
self.assertEqual(info.downloaded[fp], failure)
# ... encapsulating the original FileException ...
self.assertEqual(info.downloaded[fp].value, file_exc)
# ... but it should not store the returnValue exception on its context
# ... but it should not store the StopIteration exception on its context
context = getattr(info.downloaded[fp].value, '__context__', None)
self.assertIsNone(context)
@ -199,12 +198,19 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
pipeline_class = MockedMediaPipeline
def _callback(self, result):
self.pipe._mockcalled.append('request_callback')
return result
def _errback(self, result):
self.pipe._mockcalled.append('request_errback')
return result
@inlineCallbacks
def test_result_succeed(self):
cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
rsp = Response('http://url1')
req = Request('http://url1', meta=dict(response=rsp), callback=cb, errback=eb)
req = Request('http://url1', meta=dict(response=rsp),
callback=self._callback, errback=self._errback)
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, rsp)])
@ -215,10 +221,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
fail = Failure(Exception())
req = Request('http://url1', meta=dict(response=fail), callback=cb, errback=eb)
req = Request('http://url1', meta=dict(response=fail),
callback=self._callback, errback=self._errback)
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(False, fail)])
@ -325,8 +330,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, 'ITSME')])
self.assertEqual(self.pipe._mockcalled, \
['get_media_requests', 'media_to_download', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'item_completed'])
class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):

2
tests/test_spidermiddleware_output_chain.py
View File

@ -292,7 +292,7 @@ class TestSpiderMiddleware(TestCase):
crawler = get_crawler(spider)
with LogCapture() as log:
yield crawler.crawl(mockserver=self.mockserver)
raise defer.returnValue(log)
return log
@defer.inlineCallbacks
def test_recovery(self):

333
tests/test_spidermiddleware_referer.py
View File

@ -24,7 +24,7 @@ class TestRefererMiddleware(TestCase):
resp_headers = {}
settings = {}
scenarii = [
('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),
('http://scrapytest.org', 'http://scrapytest.org/', b'http://scrapytest.org'),
]
def setUp(self):
@ -54,57 +54,57 @@ class MixinDefault:
with some additional filtering of s3://
"""
scenarii = [
('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
('https://example.com/', 'http://scrapy.org/', None),
('https://example.com/', 'https://scrapy.org/', b'https://example.com/'),
('http://example.com/', 'http://scrapy.org/', b'http://example.com/'),
('http://example.com/', 'https://scrapy.org/', b'http://example.com/'),
('https://example.com/', 'http://scrapy.org/', None),
# no credentials leak
('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
('http://user:password@example.com/', 'https://scrapy.org/', b'http://example.com/'),
# no referrer leak for local schemes
('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'https://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
# no referrer leak for s3 origins
('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
('s3://mybucket/path/to/data.csv', 'https://scrapy.org/', None),
('s3://mybucket/path/to/data.csv', 'http://scrapy.org/', None),
]
class MixinNoReferrer:
scenarii = [
('https://example.com/page.html', 'https://example.com/', None),
('http://www.example.com/', 'https://scrapy.org/', None),
('http://www.example.com/', 'http://scrapy.org/', None),
('https://www.example.com/', 'http://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
('https://example.com/page.html', 'https://example.com/', None),
('http://www.example.com/', 'https://scrapy.org/', None),
('http://www.example.com/', 'http://scrapy.org/', None),
('https://www.example.com/', 'http://scrapy.org/', None),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', None),
]
class MixinNoReferrerWhenDowngrade:
scenarii = [
# TLS to TLS: send non-empty referrer
('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
('https://example.com/page.html', 'https://not.example.com/', b'https://example.com/page.html'),
('https://example.com/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
('https://example.com:443/page.html', 'https://scrapy.org/', b'https://example.com/page.html'),
('https://example.com:444/page.html', 'https://scrapy.org/', b'https://example.com:444/page.html'),
('ftps://example.com/urls.zip', 'https://scrapy.org/', b'ftps://example.com/urls.zip'),
# TLS to non-TLS: do not send referrer
('https://example.com/page.html', 'http://not.example.com/', None),
('https://example.com/page.html', 'http://scrapy.org/', None),
('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
('https://example.com/page.html', 'http://not.example.com/', None),
('https://example.com/page.html', 'http://scrapy.org/', None),
('ftps://example.com/urls.zip', 'http://scrapy.org/', None),
# non-TLS to TLS or non-TLS: send referrer
('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
('http://example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
('http://example.com/page.html', 'https://scrapy.org/', b'http://example.com/page.html'),
('http://example.com:8080/page.html', 'https://scrapy.org/', b'http://example.com:8080/page.html'),
('http://example.com:80/page.html', 'http://not.example.com/', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://scrapy.org/', b'http://example.com/page.html'),
('http://example.com:443/page.html', 'http://scrapy.org/', b'http://example.com:443/page.html'),
('ftp://example.com/urls.zip', 'http://scrapy.org/', b'ftp://example.com/urls.zip'),
('ftp://example.com/urls.zip', 'https://scrapy.org/', b'ftp://example.com/urls.zip'),
# test for user/password stripping
('http://user:password@example.com/page.html', 'https://not.example.com/', b'http://example.com/page.html'),
@ -114,43 +114,43 @@ class MixinNoReferrerWhenDowngrade:
class MixinSameOrigin:
scenarii = [
# Same origin (protocol, host, port): send referrer
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
# Different host: do NOT send referrer
('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
('https://example.com/page.html', 'https://not.example.com/otherpage.html', None),
('http://example.com/page.html', 'http://not.example.com/otherpage.html', None),
('http://example.com/page.html', 'http://www.example.com/otherpage.html', None),
# Different port: do NOT send referrer
('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
('https://example.com:444/page.html', 'https://example.com/not-page.html', None),
('http://example.com:81/page.html', 'http://example.com/not-page.html', None),
('http://example.com/page.html', 'http://example.com:81/not-page.html', None),
# Different protocols: do NOT send refferer
('https://example.com/page.html', 'http://example.com/not-page.html', None),
('https://example.com/page.html', 'http://not.example.com/', None),
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
('https://example.com/page.html', 'http://example.com/not-page.html', None),
('https://example.com/page.html', 'http://not.example.com/', None),
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
('ftp://example.com/urls.zip', 'http://example.com/not-page.html', None),
('ftps://example.com/urls.zip', 'https://example.com/not-page.html', None),
# test for user/password stripping
('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
('https://user:password@example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('https://user:password@example.com/page.html', 'http://example.com/not-page.html', None),
]
class MixinOrigin:
scenarii = [
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent (yes, even for downgrades)
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('https://example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
# test for user/password stripping
('https://user:password@example.com/page.html', 'http://scrapy.org', b'https://example.com/'),
@ -160,129 +160,129 @@ class MixinOrigin:
class MixinStrictOrigin:
scenarii = [
# TLS or non-TLS to TLS or non-TLS: referrer origin is sent but not for downgrades
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/'),
('https://example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('http://example.com/page.html', 'http://scrapy.org', b'http://example.com/'),
# downgrade: send nothing
('https://example.com/page.html', 'http://scrapy.org', None),
('https://example.com/page.html', 'http://scrapy.org', None),
# upgrade: send origin
('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),
('http://example.com/page.html', 'https://scrapy.org', b'http://example.com/'),
# test for user/password stripping
('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('https://user:password@example.com/page.html', 'http://scrapy.org', None),
('https://user:password@example.com/page.html', 'https://scrapy.org', b'https://example.com/'),
('https://user:password@example.com/page.html', 'http://scrapy.org', None),
]
class MixinOriginWhenCrossOrigin:
scenarii = [
# Same origin (protocol, host, port): send referrer
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
# Different host: send origin as referrer
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
# exact match required
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
# Different port: send origin as referrer
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
# Different protocols: send origin as referrer
('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('https://example4.com/page.html', 'http://example4.com/not-page.html', b'https://example4.com/'),
('https://example4.com/page.html', 'http://not.example4.com/', b'https://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
# test for user/password stripping
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
# TLS to non-TLS downgrade: send origin
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', b'https://example5.com/'),
]
class MixinStrictOriginWhenCrossOrigin:
scenarii = [
# Same origin (protocol, host, port): send referrer
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
('https://example.com/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('https://example.com:443/page.html', 'https://example.com/not-page.html', b'https://example.com/page.html'),
('http://example.com:80/page.html', 'http://example.com/not-page.html', b'http://example.com/page.html'),
('http://example.com/page.html', 'http://example.com:80/not-page.html', b'http://example.com/page.html'),
('http://example.com:8888/page.html', 'http://example.com:8888/not-page.html', b'http://example.com:8888/page.html'),
# Different host: send origin as referrer
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
('https://example2.com/page.html', 'https://scrapy.org/otherpage.html', b'https://example2.com/'),
('https://example2.com/page.html', 'https://not.example2.com/otherpage.html', b'https://example2.com/'),
('http://example2.com/page.html', 'http://not.example2.com/otherpage.html', b'http://example2.com/'),
# exact match required
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
('http://example2.com/page.html', 'http://www.example2.com/otherpage.html', b'http://example2.com/'),
# Different port: send origin as referrer
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
('https://example3.com:444/page.html', 'https://example3.com/not-page.html', b'https://example3.com:444/'),
('http://example3.com:81/page.html', 'http://example3.com/not-page.html', b'http://example3.com:81/'),
# downgrade
('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
('https://example4.com/page.html', 'http://not.example4.com/', None),
('https://example4.com/page.html', 'http://example4.com/not-page.html', None),
('https://example4.com/page.html', 'http://not.example4.com/', None),
# non-TLS to non-TLS
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
('ftp://example4.com/urls.zip', 'http://example4.com/not-page.html', b'ftp://example4.com/'),
# upgrade
('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
('http://example4.com/page.html', 'https://example4.com/not-page.html', b'http://example4.com/'),
('http://example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/'),
# Different protocols: send origin as referrer
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
('ftps://example4.com/urls.zip', 'https://example4.com/not-page.html', b'ftps://example4.com/'),
# test for user/password stripping
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
('https://user:password@example5.com/page.html', 'https://example5.com/not-page.html', b'https://example5.com/page.html'),
# TLS to non-TLS downgrade: send nothing
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
('https://user:password@example5.com/page.html', 'http://example5.com/not-page.html', None),
]
class MixinUnsafeUrl:
scenarii = [
# TLS to TLS: send referrer
('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
('https://example.com/sekrit.html', 'http://not.example.com/', b'https://example.com/sekrit.html'),
('https://example1.com/page.html', 'https://not.example1.com/', b'https://example1.com/page.html'),
('https://example1.com/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
('https://example1.com:443/page.html', 'https://scrapy.org/', b'https://example1.com/page.html'),
('https://example1.com:444/page.html', 'https://scrapy.org/', b'https://example1.com:444/page.html'),
('ftps://example1.com/urls.zip', 'https://scrapy.org/', b'ftps://example1.com/urls.zip'),
# TLS to non-TLS: send referrer (yes, it's unsafe)
('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
('https://example2.com/page.html', 'http://not.example2.com/', b'https://example2.com/page.html'),
('https://example2.com/page.html', 'http://scrapy.org/', b'https://example2.com/page.html'),
('ftps://example2.com/urls.zip', 'http://scrapy.org/', b'ftps://example2.com/urls.zip'),
# non-TLS to TLS or non-TLS: send referrer (yes, it's unsafe)
('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
('http://example3.com/page.html', 'https://not.example3.com/', b'http://example3.com/page.html'),
('http://example3.com/page.html', 'https://scrapy.org/', b'http://example3.com/page.html'),
('http://example3.com:8080/page.html', 'https://scrapy.org/', b'http://example3.com:8080/page.html'),
('http://example3.com:80/page.html', 'http://not.example3.com/', b'http://example3.com/page.html'),
('http://example3.com/page.html', 'http://scrapy.org/', b'http://example3.com/page.html'),
('http://example3.com:443/page.html', 'http://scrapy.org/', b'http://example3.com:443/page.html'),
('ftp://example3.com/urls.zip', 'http://scrapy.org/', b'ftp://example3.com/urls.zip'),
('ftp://example3.com/urls.zip', 'https://scrapy.org/', b'ftp://example3.com/urls.zip'),
# test for user/password stripping
('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),
('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),
('http://user:password@example4.com/page.html', 'https://not.example4.com/', b'http://example4.com/page.html'),
('https://user:password@example4.com/page.html', 'http://scrapy.org/', b'https://example4.com/page.html'),
]
@ -339,12 +339,12 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
class TestSettingsCustomPolicy(TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
scenarii = [
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
('http://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('https://example.com/', 'http://scrapy.org/', b'http://python.org/'),
('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'),
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
('http://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('https://example.com/', 'http://scrapy.org/', b'http://python.org/'),
('file:///home/path/to/somefile.html', 'https://scrapy.org/', b'https://python.org/'),
('file:///home/path/to/somefile.html', 'http://scrapy.org/', b'http://python.org/'),
]
@ -541,7 +541,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
scenarii = [
( 'http://scrapytest.org/1', # parent
(
'http://scrapytest.org/1', # parent
'http://scrapytest.org/2', # target
(
# redirections: code, URL
@ -551,7 +552,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
b'http://scrapytest.org/1', # expected initial referer
b'http://scrapytest.org/1', # expected referer for the redirection request
),
( 'https://scrapytest.org/1',
(
'https://scrapytest.org/1',
'https://scrapytest.org/2',
(
# redirecting to non-secure URL
@ -560,7 +562,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
b'https://scrapytest.org/1',
b'https://scrapytest.org/1',
),
( 'https://scrapytest.org/1',
(
'https://scrapytest.org/1',
'https://scrapytest.com/2',
(
# redirecting to non-secure URL: different origin
@ -602,7 +605,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
"""
settings = {'REFERRER_POLICY': 'no-referrer'}
scenarii = [
( 'http://scrapytest.org/1', # parent
(
'http://scrapytest.org/1', # parent
'http://scrapytest.org/2', # target
(
# redirections: code, URL
@ -612,7 +616,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
None, # expected initial "Referer"
None, # expected "Referer" for the redirection request
),
( 'https://scrapytest.org/1',
(
'https://scrapytest.org/1',
'https://scrapytest.org/2',
(
(301, 'http://scrapytest.org/3'),
@ -620,7 +625,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
None,
None,
),
( 'https://scrapytest.org/1',
(
'https://scrapytest.org/1',
'https://example.com/2', # different origin
(
(301, 'http://scrapytest.com/3'),
@ -641,7 +647,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
"""
settings = {'REFERRER_POLICY': 'same-origin'}
scenarii = [
( 'http://scrapytest.org/101', # origin
(
'http://scrapytest.org/101', # origin
'http://scrapytest.org/102', # target
(
# redirections: code, URL
@ -651,7 +658,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
b'http://scrapytest.org/101', # expected initial "Referer"
b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
(
'https://scrapytest.org/201',
'https://scrapytest.org/202',
(
# redirecting from secure to non-secure URL == different origin
@ -660,7 +668,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/201',
None,
),
( 'https://scrapytest.org/301',
(
'https://scrapytest.org/301',
'https://scrapytest.org/302',
(
# different domain == different origin
@ -683,7 +692,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
"""
settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
scenarii = [
( 'http://scrapytest.org/101',
(
'http://scrapytest.org/101',
'http://scrapytest.org/102',
(
(301, 'http://scrapytest.org/103'),
@ -692,7 +702,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
b'http://scrapytest.org/', # send origin
b'http://scrapytest.org/', # redirects to same origin: send origin
),
( 'https://scrapytest.org/201',
(
'https://scrapytest.org/201',
'https://scrapytest.org/202',
(
# redirecting to non-secure URL: no referrer
@ -701,7 +712,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/',
None,
),
( 'https://scrapytest.org/301',
(
'https://scrapytest.org/301',
'https://scrapytest.org/302',
(
# redirecting to non-secure URL (different domain): no referrer
@ -710,7 +722,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/',
None,
),
( 'http://scrapy.org/401',
(
'http://scrapy.org/401',
'http://example.com/402',
(
(301, 'http://scrapytest.org/403'),
@ -718,7 +731,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
b'http://scrapy.org/',
b'http://scrapy.org/',
),
( 'https://scrapy.org/501',
(
'https://scrapy.org/501',
'https://example.com/502',
(
# HTTPS all along, so origin referrer is kept as-is
@ -728,7 +742,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
b'https://scrapy.org/',
b'https://scrapy.org/',
),
( 'https://scrapytest.org/601',
(
'https://scrapytest.org/601',
'http://scrapytest.org/602', # TLS to non-TLS: no referrer
(
(301, 'https://scrapytest.org/603'), # TLS URL again: (still) no referrer
@ -750,7 +765,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
"""
settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
scenarii = [
( 'http://scrapytest.org/101', # origin
(
'http://scrapytest.org/101', # origin
'http://scrapytest.org/102', # target + redirection
(
# redirections: code, URL
@ -760,7 +776,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'http://scrapytest.org/101', # expected initial referer
b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
(
'https://scrapytest.org/201',
'https://scrapytest.org/202',
(
# redirecting to non-secure URL: send origin
@ -769,7 +786,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/201',
b'https://scrapytest.org/',
),
( 'https://scrapytest.org/301',
(
'https://scrapytest.org/301',
'https://scrapytest.org/302',
(
# redirecting to non-secure URL (different domain): send origin
@ -778,7 +796,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/301',
b'https://scrapytest.org/',
),
( 'http://scrapy.org/401',
(
'http://scrapy.org/401',
'http://example.com/402',
(
(301, 'http://scrapytest.org/403'),
@ -786,7 +805,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'http://scrapy.org/',
b'http://scrapy.org/',
),
( 'https://scrapy.org/501',
(
'https://scrapy.org/501',
'https://example.com/502',
(
# all different domains: send origin
@ -796,7 +816,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'https://scrapy.org/',
b'https://scrapy.org/',
),
( 'https://scrapytest.org/301',
(
'https://scrapytest.org/301',
'http://scrapytest.org/302', # TLS to non-TLS: send origin
(
(301, 'https://scrapytest.org/303'), # TLS URL again: send origin (also)
@ -820,7 +841,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
"""
settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
scenarii = [
( 'http://scrapytest.org/101', # origin
(
'http://scrapytest.org/101', # origin
'http://scrapytest.org/102', # target + redirection
(
# redirections: code, URL
@ -830,7 +852,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'http://scrapytest.org/101', # expected initial referer
b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
(
'https://scrapytest.org/201',
'https://scrapytest.org/202',
(
# redirecting to non-secure URL: do not send the "Referer" header
@ -839,7 +862,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/201',
None,
),
( 'https://scrapytest.org/301',
(
'https://scrapytest.org/301',
'https://scrapytest.org/302',
(
# redirecting to non-secure URL (different domain): send origin
@ -848,7 +872,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'https://scrapytest.org/301',
None,
),
( 'http://scrapy.org/401',
(
'http://scrapy.org/401',
'http://example.com/402',
(
(301, 'http://scrapytest.org/403'),
@ -856,7 +881,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'http://scrapy.org/',
b'http://scrapy.org/',
),
( 'https://scrapy.org/501',
(
'https://scrapy.org/501',
'https://example.com/502',
(
# all different domains: send origin
@ -866,7 +892,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
b'https://scrapy.org/',
b'https://scrapy.org/',
),
( 'https://scrapytest.org/601',
(
'https://scrapytest.org/601',
'http://scrapytest.org/602', # TLS to non-TLS: do not send "Referer"
(
(301, 'https://scrapytest.org/603'), # TLS URL again: (still) send nothing

52
tests/test_utils_iterators.py
View File

@ -157,7 +157,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, 'product')
self.assertRaises(AssertionError, next, i)
self.assertRaises(TypeError, next, i)
def test_xmliter_encoding(self):
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
@ -250,10 +250,10 @@ class UtilsCsvTestCase(unittest.TestCase):
result = [row for row in csv]
self.assertEqual(result,
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
@ -266,10 +266,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter='\t')
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
def test_csviter_quotechar(self):
body1 = get_testdata('feeds', 'feed-sample6.csv')
@ -279,19 +279,19 @@ class UtilsCsvTestCase(unittest.TestCase):
csv1 = csviter(response1, quotechar="'")
self.assertEqual([row for row in csv1],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
response2 = TextResponse(url="http://example.com/", body=body2)
csv2 = csviter(response2, delimiter="|", quotechar="'")
self.assertEqual([row for row in csv2],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
def test_csviter_wrong_quotechar(self):
body = get_testdata('feeds', 'feed-sample6.csv')
@ -299,10 +299,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
{u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
{u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
{u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
[{u"'id'": u"1", u"'name'": u"'alpha'", u"'value'": u"'foobar'"},
{u"'id'": u"2", u"'name'": u"'unicode'", u"'value'": u"'\xfan\xedc\xf3d\xe9\u203d'"},
{u"'id'": u"'3'", u"'name'": u"'multi'", u"'value'": u"'foo"},
{u"'id'": u"4", u"'name'": u"'empty'", u"'value'": u""}])
def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
@ -310,10 +310,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, delimiter='\t')
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
def test_csviter_headers(self):
sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
@ -323,10 +323,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response, headers=[h.decode('utf-8') for h in headers])
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'},
{u'id': u'4', u'name': u'empty', u'value': u''}])
def test_csviter_falserow(self):
body = get_testdata('feeds', 'feed-sample3.csv')
@ -336,10 +336,10 @@ class UtilsCsvTestCase(unittest.TestCase):
csv = csviter(response)
self.assertEqual([row for row in csv],
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
[{u'id': u'1', u'name': u'alpha', u'value': u'foobar'},
{u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
{u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL},
{u'id': u'4', u'name': u'empty', u'value': u''}])
def test_csviter_exception(self):
body = get_testdata('feeds', 'feed-sample3.csv')

4
tests/test_utils_python.py
View File

@ -145,7 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase):
get_z = operator.itemgetter('z')
get_meta = operator.attrgetter('meta')
compare_z = lambda obj: get_z(get_meta(obj))
def compare_z(obj):
return get_z(get_meta(obj))
self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
# fail z equality

92
tests/test_utils_reqser.py
View File

@ -2,7 +2,7 @@ import unittest
from scrapy.http import Request, FormRequest
from scrapy.spiders import Spider
from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
from scrapy.utils.reqser import request_to_dict, request_from_dict
class RequestSerializationTest(unittest.TestCase):
@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase):
errback=self.spider.handle_error)
self._assert_serializes_ok(r, spider=self.spider)
def test_reference_callback_serialization(self):
r = Request("http://www.example.com",
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
self.assertEqual(request_dict['callback'], 'parse_item_reference')
self.assertEqual(request_dict['errback'], 'handle_error_reference')
def test_private_reference_callback_serialization(self):
r = Request("http://www.example.com",
callback=self.spider._TestSpider__parse_item_reference,
errback=self.spider._TestSpider__handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
self.assertEqual(request_dict['callback'],
'_TestSpider__parse_item_reference')
self.assertEqual(request_dict['errback'],
'_TestSpider__handle_error_reference')
def test_private_callback_serialization(self):
r = Request("http://www.example.com",
callback=self.spider._TestSpider__parse_item_private,
@ -81,41 +101,6 @@ class RequestSerializationTest(unittest.TestCase):
errback=self.spider.handle_error)
self._assert_serializes_ok(r, spider=self.spider)
def test_private_callback_name_matching(self):
self.assertTrue(_is_private_method('__a'))
self.assertTrue(_is_private_method('__a_'))
self.assertTrue(_is_private_method('__a_a'))
self.assertTrue(_is_private_method('__a_a_'))
self.assertTrue(_is_private_method('__a__a'))
self.assertTrue(_is_private_method('__a__a_'))
self.assertTrue(_is_private_method('__a___a'))
self.assertTrue(_is_private_method('__a___a_'))
self.assertTrue(_is_private_method('___a'))
self.assertTrue(_is_private_method('___a_'))
self.assertTrue(_is_private_method('___a_a'))
self.assertTrue(_is_private_method('___a_a_'))
self.assertTrue(_is_private_method('____a_a_'))
self.assertFalse(_is_private_method('_a'))
self.assertFalse(_is_private_method('_a_'))
self.assertFalse(_is_private_method('__a__'))
self.assertFalse(_is_private_method('__'))
self.assertFalse(_is_private_method('___'))
self.assertFalse(_is_private_method('____'))
def _assert_mangles_to(self, obj, name):
func = getattr(obj, name)
self.assertEqual(
_mangle_private_name(obj, func, func.__name__),
name
)
def test_private_name_mangling(self):
self._assert_mangles_to(
self.spider, '_TestSpider__parse_item_private')
self._assert_mangles_to(
self.spider, '_TestSpiderMixin__mixin_callback')
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
self.assertRaises(ValueError, request_to_dict, r)
@ -125,14 +110,49 @@ class RequestSerializationTest(unittest.TestCase):
r = Request("http://www.example.com", callback=self.spider.parse_item)
self.assertRaises(ValueError, request_to_dict, r)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
name = 'my_spider'
def parse(self, response):
pass
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
setattr(spider, 'parse', None)
self.assertRaises(ValueError, request_to_dict, r, spider=spider)
class TestSpiderMixin:
def __mixin_callback(self, response):
pass
def parse_item(response):
pass
def handle_error(failure):
pass
def private_parse_item(response):
pass
def private_handle_error(failure):
pass
class TestSpider(Spider, TestSpiderMixin):
name = 'test'
parse_item_reference = parse_item
handle_error_reference = handle_error
__parse_item_reference = private_parse_item
__handle_error_reference = private_handle_error
def parse_item(self, response):
pass

4
tests/test_utils_signal.py
View File

@ -90,8 +90,10 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
class SendCatchLogTest2(unittest.TestCase):
def test_error_logged_if_deferred_not_supported(self):
def test_handler():
return defer.Deferred()
test_signal = object()
test_handler = lambda: defer.Deferred()
dispatcher.connect(test_handler, test_signal)
with LogCapture() as l:
send_catch_log(test_signal)

40
tests/test_utils_url.py
View File

@ -203,29 +203,29 @@ def create_skipped_scheme_t(args):
for k, args in enumerate([
('/index', 'file://'),
('/index.html', 'file://'),
('./index.html', 'file://'),
('../index.html', 'file://'),
('../../index.html', 'file://'),
('./data/index.html', 'file://'),
('.hidden/data/index.html', 'file://'),
('/home/user/www/index.html', 'file://'),
('//home/user/www/index.html', 'file://'),
('file:///home/user/www/index.html', 'file://'),
('/index', 'file://'),
('/index.html', 'file://'),
('./index.html', 'file://'),
('../index.html', 'file://'),
('../../index.html', 'file://'),
('./data/index.html', 'file://'),
('.hidden/data/index.html', 'file://'),
('/home/user/www/index.html', 'file://'),
('//home/user/www/index.html', 'file://'),
('file:///home/user/www/index.html', 'file://'),
('index.html', 'http://'),
('example.com', 'http://'),
('www.example.com', 'http://'),
('www.example.com/index.html', 'http://'),
('http://example.com', 'http://'),
('http://example.com/index.html', 'http://'),
('localhost', 'http://'),
('localhost/index.html', 'http://'),
('index.html', 'http://'),
('example.com', 'http://'),
('www.example.com', 'http://'),
('www.example.com/index.html', 'http://'),
('http://example.com', 'http://'),
('http://example.com/index.html', 'http://'),
('localhost', 'http://'),
('localhost/index.html', 'http://'),
# some corner cases (default to http://)
('/', 'http://'),
('.../test', 'http://'),
('/', 'http://'),
('.../test', 'http://'),
], start=1):
t_method = create_guess_scheme_t(args)

30
tests/test_webclient.py
View File

@ -53,28 +53,28 @@ class ParseUrlTestCase(unittest.TestCase):
def testParse(self):
lip = '127.0.0.1'
tests = (
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
)
for url, test in tests:

18
tox.ini
View File

@ -74,11 +74,15 @@ deps =
changedir = docs
deps =
-rdocs/requirements.txt
setenv =
READTHEDOCS_PROJECT=scrapy
READTHEDOCS_VERSION=master
[testenv:docs]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b html . {envtmpdir}/html
@ -86,6 +90,7 @@ commands =
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -b coverage . {envtmpdir}/coverage
@ -93,19 +98,10 @@ commands =
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
[asyncio]
[testenv:asyncio]
commands =
{[testenv]commands} --reactor=asyncio
[testenv:py35-asyncio]
basepython = python3.5
deps = {[testenv]deps}
commands = {[asyncio]commands}
[testenv:py38-asyncio]
basepython = python3.8
deps = {[testenv]deps}
commands = {[asyncio]commands}