mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into flake8-max-line-length
This commit is contained in:
Eugenio Lacuesta
commit
7926da4bd0
|
|
@ -12,6 +12,8 @@ collect_ignore = [
|
|||
"scrapy/utils/testsite.py",
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
||||
*_py_files("tests/CrawlerProcess"),
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
|
||||
*_py_files("tests/CrawlerRunner"),
|
||||
# Py36-only parts of respective tests
|
||||
*_py_files("tests/py36"),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -295,3 +295,10 @@ intersphinx_mapping = {
|
|||
# ------------------------------------
|
||||
|
||||
hoverxref_auto_ref = True
|
||||
hoverxref_role_types = {
|
||||
"class": "tooltip",
|
||||
"confval": "tooltip",
|
||||
"hoverxref": "tooltip",
|
||||
"mod": "tooltip",
|
||||
"ref": "tooltip",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,16 +25,16 @@ Scrapy.
|
|||
If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
|
||||
|
||||
If you're new to programming and want to start with Python, the following books
|
||||
may be useful to you:
|
||||
may be useful to you:
|
||||
|
||||
* `Automate the Boring Stuff With Python`_
|
||||
|
||||
* `How To Think Like a Computer Scientist`_
|
||||
* `How To Think Like a Computer Scientist`_
|
||||
|
||||
* `Learn Python 3 The Hard Way`_
|
||||
* `Learn Python 3 The Hard Way`_
|
||||
|
||||
You can also take a look at `this list of Python resources for non-programmers`_,
|
||||
as well as the `suggested resources in the learnpython-subreddit`_.
|
||||
as well as the `suggested resources in the learnpython-subreddit`_.
|
||||
|
||||
.. _Python: https://www.python.org/
|
||||
.. _this list of Python resources for non-programmers: https://wiki.python.org/moin/BeginnersGuide/NonProgrammers
|
||||
|
|
@ -62,7 +62,7 @@ This will create a ``tutorial`` directory with the following contents::
|
|||
__init__.py
|
||||
|
||||
items.py # project items definition file
|
||||
|
||||
|
||||
middlewares.py # project middlewares file
|
||||
|
||||
pipelines.py # project pipelines file
|
||||
|
|
@ -287,8 +287,8 @@ to be scraped, you can at least get **some** data.
|
|||
|
||||
Besides the :meth:`~scrapy.selector.SelectorList.getall` and
|
||||
:meth:`~scrapy.selector.SelectorList.get` methods, you can also use
|
||||
the :meth:`~scrapy.selector.SelectorList.re` method to extract using `regular
|
||||
expressions`_:
|
||||
the :meth:`~scrapy.selector.SelectorList.re` method to extract using
|
||||
:doc:`regular expressions <library/re>`:
|
||||
|
||||
>>> response.css('title::text').re(r'Quotes.*')
|
||||
['Quotes to Scrape']
|
||||
|
|
@ -305,7 +305,6 @@ with a selector (see :ref:`topics-developer-tools`).
|
|||
`Selector Gadget`_ is also a nice tool to quickly find CSS selector for
|
||||
visually selected elements, which works in many browsers.
|
||||
|
||||
.. _regular expressions: https://docs.python.org/3/library/re.html
|
||||
.. _Selector Gadget: https://selectorgadget.com/
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
Sphinx>=2.1
|
||||
sphinx-hoverxref
|
||||
sphinx-notfound-page
|
||||
sphinx_rtd_theme
|
||||
Sphinx>=3.0
|
||||
sphinx-hoverxref>=0.2b1
|
||||
sphinx-notfound-page>=0.4
|
||||
sphinx_rtd_theme>=0.4
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ Detecting check runs
|
|||
====================
|
||||
|
||||
When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is
|
||||
set to the ``true`` string. You can use `os.environ`_ to perform any change to
|
||||
set to the ``true`` string. You can use :data:`os.environ` to perform any change to
|
||||
your spiders or your settings when ``scrapy check`` is used::
|
||||
|
||||
import os
|
||||
|
|
@ -148,5 +148,3 @@ your spiders or your settings when ``scrapy check`` is used::
|
|||
def __init__(self):
|
||||
if os.environ.get('SCRAPY_CHECK'):
|
||||
pass # Do some scraper adjustments when a check is running
|
||||
|
||||
.. _os.environ: https://docs.python.org/3/library/os.html#os.environ
|
||||
|
|
|
|||
|
|
@ -76,8 +76,8 @@ becomes::
|
|||
|
||||
Coroutines may be used to call asynchronous code. This includes other
|
||||
coroutines, functions that return Deferreds and functions that return
|
||||
`awaitable objects`_ such as :class:`~asyncio.Future`. This means you can use
|
||||
many useful Python libraries providing such code::
|
||||
:term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
|
||||
This means you can use many useful Python libraries providing such code::
|
||||
|
||||
class MySpider(Spider):
|
||||
# ...
|
||||
|
|
@ -107,4 +107,3 @@ Common use cases for asynchronous code include:
|
|||
:ref:`the screenshot pipeline example<ScreenshotPipeline>`).
|
||||
|
||||
.. _aio-libs: https://github.com/aio-libs
|
||||
.. _awaitable objects: https://docs.python.org/3/glossary.html#term-awaitable
|
||||
|
|
|
|||
|
|
@ -292,6 +292,9 @@ Alternatively, if you want to know the arguments needed to recreate that
|
|||
request you can use the :func:`scrapy.utils.curl.curl_to_request_kwargs`
|
||||
function to get a dictionary with the equivalent arguments.
|
||||
|
||||
Note that to translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
||||
As you can see, with a few inspections in the `Network`-tool we
|
||||
were able to easily replicate the dynamic requests of the scrolling
|
||||
functionality of the page. Crawling dynamic pages can be quite
|
||||
|
|
|
|||
|
|
@ -739,7 +739,7 @@ HttpProxyMiddleware
|
|||
This middleware sets the HTTP proxy to use for requests, by setting the
|
||||
``proxy`` meta value for :class:`~scrapy.http.Request` objects.
|
||||
|
||||
Like the Python standard library modules `urllib`_ and `urllib2`_, it obeys
|
||||
Like the Python standard library module :mod:`urllib.request`, it obeys
|
||||
the following environment variables:
|
||||
|
||||
* ``http_proxy``
|
||||
|
|
@ -751,9 +751,6 @@ HttpProxyMiddleware
|
|||
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
|
||||
environment variables, and it will also ignore ``no_proxy`` environment variable.
|
||||
|
||||
.. _urllib: https://docs.python.org/2/library/urllib.html
|
||||
.. _urllib2: https://docs.python.org/2/library/urllib2.html
|
||||
|
||||
RedirectMiddleware
|
||||
------------------
|
||||
|
||||
|
|
@ -829,6 +826,7 @@ REDIRECT_MAX_TIMES
|
|||
Default: ``20``
|
||||
|
||||
The maximum number of redirections that will be followed for a single request.
|
||||
After this maximum, the request's response is returned as is.
|
||||
|
||||
MetaRefreshMiddleware
|
||||
---------------------
|
||||
|
|
@ -1036,8 +1034,7 @@ Scrapy uses this parser by default.
|
|||
RobotFileParser
|
||||
~~~~~~~~~~~~~~~
|
||||
|
||||
Based on `RobotFileParser
|
||||
<https://docs.python.org/3.7/library/urllib.robotparser.html>`_:
|
||||
Based on :class:`~urllib.robotparser.RobotFileParser`:
|
||||
|
||||
* is Python's built-in robots.txt_ parser
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,9 @@ If you get the expected response `sometimes`, but not always, the issue is
|
|||
probably not your request, but the target server. The target server might be
|
||||
buggy, overloaded, or :ref:`banning <bans>` some of your requests.
|
||||
|
||||
Note that to translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
||||
.. _topics-handling-response-formats:
|
||||
|
||||
Handling different response formats
|
||||
|
|
@ -115,7 +118,7 @@ data from it depends on the type of response:
|
|||
- If the response is HTML or XML, use :ref:`selectors
|
||||
<topics-selectors>` as usual.
|
||||
|
||||
- If the response is JSON, use `json.loads`_ to load the desired data from
|
||||
- If the response is JSON, use :func:`json.loads` to load the desired data from
|
||||
:attr:`response.text <scrapy.http.TextResponse.text>`::
|
||||
|
||||
data = json.loads(response.text)
|
||||
|
|
@ -130,8 +133,9 @@ data from it depends on the type of response:
|
|||
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
||||
containing the desired data, see :ref:`topics-parsing-javascript`.
|
||||
|
||||
- If the response is CSS, use a `regular expression`_ to extract the desired
|
||||
data from :attr:`response.text <scrapy.http.TextResponse.text>`.
|
||||
- If the response is CSS, use a :doc:`regular expression <library/re>` to
|
||||
extract the desired data from
|
||||
:attr:`response.text <scrapy.http.TextResponse.text>`.
|
||||
|
||||
.. _topics-parsing-images:
|
||||
|
||||
|
|
@ -168,8 +172,9 @@ JavaScript code:
|
|||
Once you have a string with the JavaScript code, you can extract the desired
|
||||
data from it:
|
||||
|
||||
- You might be able to use a `regular expression`_ to extract the desired
|
||||
data in JSON format, which you can then parse with `json.loads`_.
|
||||
- You might be able to use a :doc:`regular expression <library/re>` to
|
||||
extract the desired data in JSON format, which you can then parse with
|
||||
:func:`json.loads`.
|
||||
|
||||
For example, if the JavaScript code contains a separate line like
|
||||
``var data = {"field": "value"};`` you can extract that data as follows:
|
||||
|
|
@ -241,9 +246,7 @@ along with `scrapy-selenium`_ for seamless integration.
|
|||
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
|
||||
.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
|
||||
.. _js2xml: https://github.com/scrapinghub/js2xml
|
||||
.. _json.loads: https://docs.python.org/3/library/json.html#json.loads
|
||||
.. _pytesseract: https://github.com/madmaze/pytesseract
|
||||
.. _regular expression: https://docs.python.org/3/library/re.html
|
||||
.. _scrapy-selenium: https://github.com/clemfromspace/scrapy-selenium
|
||||
.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash
|
||||
.. _Selenium: https://www.selenium.dev/
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ Sending e-mail
|
|||
.. module:: scrapy.mail
|
||||
:synopsis: Email sending facility
|
||||
|
||||
Although Python makes sending e-mails relatively easy via the `smtplib`_
|
||||
Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
|
||||
library, Scrapy provides its own facility for sending e-mails which is very
|
||||
easy to use and it's implemented using :doc:`Twisted non-blocking IO
|
||||
<twisted:core/howto/defer-intro>`, to avoid interfering with the non-blocking
|
||||
|
|
@ -15,8 +15,6 @@ IO of the crawler. It also provides a simple API for sending attachments and
|
|||
it's very easy to configure, with a few :ref:`settings
|
||||
<topics-email-settings>`.
|
||||
|
||||
.. _smtplib: https://docs.python.org/2/library/smtplib.html
|
||||
|
||||
Quick example
|
||||
=============
|
||||
|
||||
|
|
|
|||
|
|
@ -311,7 +311,7 @@ CsvItemExporter
|
|||
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
|
||||
`csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
|
||||
:func:`csv.writer` function, so you can use any :func:`csv.writer` function
|
||||
argument to customize this exporter.
|
||||
|
||||
A typical output of this exporter would be::
|
||||
|
|
@ -320,8 +320,6 @@ CsvItemExporter
|
|||
Color TV,1200
|
||||
DVD player,200
|
||||
|
||||
.. _csv.writer: https://docs.python.org/2/library/csv.html#csv.writer
|
||||
|
||||
PickleItemExporter
|
||||
------------------
|
||||
|
||||
|
|
@ -335,15 +333,13 @@ PickleItemExporter
|
|||
:param protocol: The pickle protocol to use.
|
||||
:type protocol: int
|
||||
|
||||
For more information, refer to the `pickle module documentation`_.
|
||||
For more information, see :mod:`pickle`.
|
||||
|
||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||
:class:`BaseItemExporter` ``__init__`` method.
|
||||
|
||||
Pickle isn't a human readable format, so no output examples are provided.
|
||||
|
||||
.. _pickle module documentation: https://docs.python.org/2/library/pickle.html
|
||||
|
||||
PprintItemExporter
|
||||
------------------
|
||||
|
||||
|
|
@ -372,8 +368,8 @@ JsonItemExporter
|
|||
Exports Items in JSON format to the specified file-like object, writing all
|
||||
objects as a list of objects. The additional ``__init__`` method arguments are
|
||||
passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
|
||||
arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
|
||||
`JSONEncoder`_ ``__init__`` method argument to customize this exporter.
|
||||
arguments to the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
|
||||
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -393,8 +389,6 @@ JsonItemExporter
|
|||
stream-friendly format, consider using :class:`JsonLinesItemExporter`
|
||||
instead, or splitting the output in multiple chunks.
|
||||
|
||||
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
|
||||
|
||||
JsonLinesItemExporter
|
||||
---------------------
|
||||
|
||||
|
|
@ -403,8 +397,8 @@ JsonLinesItemExporter
|
|||
Exports Items in JSON format to the specified file-like object, writing one
|
||||
JSON-encoded item per line. The additional ``__init__`` method arguments are passed
|
||||
to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
|
||||
the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
|
||||
``__init__`` method argument to customize this exporter.
|
||||
the :class:`~json.JSONEncoder` ``__init__`` method, so you can use any
|
||||
:class:`~json.JSONEncoder` ``__init__`` method argument to customize this exporter.
|
||||
|
||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||
|
|
@ -417,8 +411,6 @@ JsonLinesItemExporter
|
|||
Unlike the one produced by :class:`JsonItemExporter`, the format produced by
|
||||
this exporter is well suited for serializing large amounts of data.
|
||||
|
||||
.. _JSONEncoder: https://docs.python.org/2/library/json.html#json.JSONEncoder
|
||||
|
||||
MarshalItemExporter
|
||||
-------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -364,7 +364,7 @@ Debugger extension
|
|||
|
||||
.. class:: Debugger
|
||||
|
||||
Invokes a `Python debugger`_ inside a running Scrapy process when a `SIGUSR2`_
|
||||
Invokes a :doc:`Python debugger <library/pdb>` inside a running Scrapy process when a `SIGUSR2`_
|
||||
signal is received. After the debugger is exited, the Scrapy process continues
|
||||
running normally.
|
||||
|
||||
|
|
@ -372,5 +372,4 @@ For more info see `Debugging in Python`_.
|
|||
|
||||
This extension only works on POSIX-compliant platforms (i.e. not Windows).
|
||||
|
||||
.. _Python debugger: https://docs.python.org/2/library/pdb.html
|
||||
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
|
||||
|
|
|
|||
|
|
@ -15,8 +15,8 @@ especially in a larger project with many spiders.
|
|||
|
||||
To define common output data format Scrapy provides the :class:`Item` class.
|
||||
:class:`Item` objects are simple containers used to collect the scraped data.
|
||||
They provide a `dictionary-like`_ API with a convenient syntax for declaring
|
||||
their available fields.
|
||||
They provide an API similar to :class:`dict` API with a convenient syntax
|
||||
for declaring their available fields.
|
||||
|
||||
Various Scrapy components use extra information provided by Items:
|
||||
exporters look at declared fields to figure out columns to export,
|
||||
|
|
@ -24,8 +24,6 @@ serialization can be customized using Item fields metadata, :mod:`trackref`
|
|||
tracks Item instances to help find memory leaks
|
||||
(see :ref:`topics-leaks-trackrefs`), etc.
|
||||
|
||||
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
||||
.. _topics-items-declaring:
|
||||
|
||||
Declaring Items
|
||||
|
|
@ -79,7 +77,7 @@ Working with Items
|
|||
|
||||
Here are some examples of common tasks performed with items, using the
|
||||
``Product`` item :ref:`declared above <topics-items-declaring>`. You will
|
||||
notice the API is very similar to the `dict API`_.
|
||||
notice the API is very similar to the :class:`dict` API.
|
||||
|
||||
Creating items
|
||||
--------------
|
||||
|
|
@ -145,7 +143,7 @@ KeyError: 'Product does not support field: lala'
|
|||
Accessing all populated values
|
||||
------------------------------
|
||||
|
||||
To access all populated values, just use the typical `dict API`_:
|
||||
To access all populated values, just use the typical :class:`dict` API:
|
||||
|
||||
>>> product.keys()
|
||||
['price', 'name']
|
||||
|
|
@ -162,11 +160,9 @@ Copying items
|
|||
To copy an item, you must first decide whether you want a shallow copy or a
|
||||
deep copy.
|
||||
|
||||
If your item contains mutable_ values like lists or dictionaries, a shallow
|
||||
copy will keep references to the same mutable values across all different
|
||||
copies.
|
||||
|
||||
.. _mutable: https://docs.python.org/3/glossary.html#term-mutable
|
||||
If your item contains :term:`mutable` values like lists or dictionaries,
|
||||
a shallow copy will keep references to the same mutable values across all
|
||||
different copies.
|
||||
|
||||
For example, if you have an item with a list of tags, and you create a shallow
|
||||
copy of that item, both the original item and the copy have the same list of
|
||||
|
|
@ -175,9 +171,7 @@ other item as well.
|
|||
|
||||
If that is not the desired behavior, use a deep copy instead.
|
||||
|
||||
See the `documentation of the copy module`_ for more information.
|
||||
|
||||
.. _documentation of the copy module: https://docs.python.org/3/library/copy.html
|
||||
See :mod:`copy` for more information.
|
||||
|
||||
To create a shallow copy of an item, you can either call
|
||||
:meth:`~scrapy.item.Item.copy` on an existing item
|
||||
|
|
@ -235,8 +229,8 @@ Item objects
|
|||
|
||||
Return a new Item optionally initialized from the given argument.
|
||||
|
||||
Items replicate the standard `dict API`_, including its ``__init__`` method, and
|
||||
also provide the following additional API members:
|
||||
Items replicate the standard :class:`dict` API, including its ``__init__``
|
||||
method, and also provide the following additional API members:
|
||||
|
||||
.. automethod:: copy
|
||||
|
||||
|
|
@ -249,22 +243,17 @@ Item objects
|
|||
:class:`Field` objects used in the :ref:`Item declaration
|
||||
<topics-items-declaring>`.
|
||||
|
||||
.. _dict API: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
||||
Field objects
|
||||
=============
|
||||
|
||||
.. class:: Field([arg])
|
||||
|
||||
The :class:`Field` class is just an alias to the built-in `dict`_ class and
|
||||
The :class:`Field` class is just an alias to the built-in :class:`dict` class and
|
||||
doesn't provide any extra functionality or attributes. In other words,
|
||||
:class:`Field` objects are plain-old Python dicts. A separate class is used
|
||||
to support the :ref:`item declaration syntax <topics-items-declaring>`
|
||||
based on class attributes.
|
||||
|
||||
.. _dict: https://docs.python.org/2/library/stdtypes.html#dict
|
||||
|
||||
|
||||
Other classes related to Item
|
||||
=============================
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,7 @@ Logging
|
|||
explicit calls to the Python standard logging. Keep reading to learn more
|
||||
about the new logging system.
|
||||
|
||||
Scrapy uses `Python's builtin logging system
|
||||
<https://docs.python.org/3/library/logging.html>`_ for event logging. We'll
|
||||
Scrapy uses :mod:`logging` for event logging. We'll
|
||||
provide some simple examples to get you started, but for more advanced
|
||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||
|
||||
|
|
@ -83,10 +82,10 @@ path::
|
|||
|
||||
.. seealso::
|
||||
|
||||
Module logging, `HowTo <https://docs.python.org/2/howto/logging.html>`_
|
||||
Module logging, :doc:`HowTo <howto/logging>`
|
||||
Basic Logging Tutorial
|
||||
|
||||
Module logging, `Loggers <https://docs.python.org/2/library/logging.html#logger-objects>`_
|
||||
Module logging, :ref:`Loggers <logger>`
|
||||
Further documentation on loggers
|
||||
|
||||
.. _topics-logging-from-spiders:
|
||||
|
|
@ -165,14 +164,12 @@ possible levels listed in :ref:`topics-logging-levels`.
|
|||
|
||||
:setting:`LOG_FORMAT` and :setting:`LOG_DATEFORMAT` specify formatting strings
|
||||
used as layouts for all messages. Those strings can contain any placeholders
|
||||
listed in `logging's logrecord attributes docs
|
||||
<https://docs.python.org/2/library/logging.html#logrecord-attributes>`_ and
|
||||
`datetime's strftime and strptime directives
|
||||
<https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior>`_
|
||||
listed in :ref:`logging's logrecord attributes docs <logrecord-attributes>` and
|
||||
:ref:`datetime's strftime and strptime directives <strftime-strptime-behavior>`
|
||||
respectively.
|
||||
|
||||
If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
|
||||
component that prints the log. It is unset by default, hence logs contain the
|
||||
component that prints the log. It is unset by default, hence logs contain the
|
||||
Scrapy component responsible for that log output.
|
||||
|
||||
Command-line options
|
||||
|
|
@ -190,7 +187,7 @@ to override some of the Scrapy settings regarding logging.
|
|||
|
||||
.. seealso::
|
||||
|
||||
Module `logging.handlers <https://docs.python.org/2/library/logging.handlers.html>`_
|
||||
Module :mod:`logging.handlers`
|
||||
Further documentation on available handlers
|
||||
|
||||
.. _custom-log-formats:
|
||||
|
|
@ -201,7 +198,7 @@ Custom Log Formats
|
|||
A custom log format can be set for different actions by extending
|
||||
:class:`~scrapy.logformatter.LogFormatter` class and making
|
||||
:setting:`LOG_FORMATTER` point to your new class.
|
||||
|
||||
|
||||
.. autoclass:: scrapy.logformatter.LogFormatter
|
||||
:members:
|
||||
|
||||
|
|
@ -256,10 +253,10 @@ scrapy.utils.log module
|
|||
In that case, its usage is not required but it's recommended.
|
||||
|
||||
Another option when running custom scripts is to manually configure the logging.
|
||||
To do this you can use `logging.basicConfig()`_ to set a basic root handler.
|
||||
To do this you can use :func:`logging.basicConfig` to set a basic root handler.
|
||||
|
||||
Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
|
||||
so it is recommended to only use `logging.basicConfig()`_ together with
|
||||
so it is recommended to only use :func:`logging.basicConfig` together with
|
||||
:class:`~scrapy.crawler.CrawlerRunner`.
|
||||
|
||||
This is an example on how to redirect ``INFO`` or higher messages to a file::
|
||||
|
|
@ -275,7 +272,3 @@ scrapy.utils.log module
|
|||
|
||||
Refer to :ref:`run-from-script` for more details about using Scrapy this
|
||||
way.
|
||||
|
||||
.. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ Request objects
|
|||
:type url: string
|
||||
|
||||
:param callback: the function that will be called with the response of this
|
||||
request (once its downloaded) as its first parameter. For more information
|
||||
request (once it's downloaded) as its first parameter. For more information
|
||||
see :ref:`topics-request-response-ref-request-callback-arguments` below.
|
||||
If a Request doesn't specify a callback, the spider's
|
||||
:meth:`~scrapy.spiders.Spider.parse` method will be used.
|
||||
|
|
@ -174,9 +174,9 @@ Request objects
|
|||
See :ref:`topics-request-meta` for a list of special meta keys
|
||||
recognized by Scrapy.
|
||||
|
||||
This dict is `shallow copied`_ when the request is cloned using the
|
||||
``copy()`` or ``replace()`` methods, and can also be accessed, in your
|
||||
spider, from the ``response.meta`` attribute.
|
||||
This dict is :doc:`shallow copied <library/copy>` when the request is
|
||||
cloned using the ``copy()`` or ``replace()`` methods, and can also be
|
||||
accessed, in your spider, from the ``response.meta`` attribute.
|
||||
|
||||
.. attribute:: Request.cb_kwargs
|
||||
|
||||
|
|
@ -185,11 +185,9 @@ Request objects
|
|||
for new Requests, which means by default callbacks only get a :class:`Response`
|
||||
object as argument.
|
||||
|
||||
This dict is `shallow copied`_ when the request is cloned using the
|
||||
``copy()`` or ``replace()`` methods, and can also be accessed, in your
|
||||
spider, from the ``response.cb_kwargs`` attribute.
|
||||
|
||||
.. _shallow copied: https://docs.python.org/2/library/copy.html
|
||||
This dict is :doc:`shallow copied <library/copy>` when the request is
|
||||
cloned using the ``copy()`` or ``replace()`` methods, and can also be
|
||||
accessed, in your spider, from the ``response.cb_kwargs`` attribute.
|
||||
|
||||
.. method:: Request.copy()
|
||||
|
||||
|
|
@ -566,12 +564,10 @@ dealing with JSON requests.
|
|||
set to ``'POST'`` automatically.
|
||||
:type data: JSON serializable object
|
||||
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying `json.dumps`_ method which is used to serialize
|
||||
:param dumps_kwargs: Parameters that will be passed to underlying :func:`json.dumps` method which is used to serialize
|
||||
data into JSON format.
|
||||
:type dumps_kwargs: dict
|
||||
|
||||
.. _json.dumps: https://docs.python.org/3/library/json.html#json.dumps
|
||||
|
||||
JsonRequest usage example
|
||||
-------------------------
|
||||
|
||||
|
|
@ -620,6 +616,9 @@ Response objects
|
|||
:param certificate: an object representing the server's SSL certificate.
|
||||
:type certificate: twisted.internet.ssl.Certificate
|
||||
|
||||
:param ip_address: The IP address of the server from which the Response originated.
|
||||
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
|
||||
|
||||
.. attribute:: Response.url
|
||||
|
||||
A string containing the URL of the response.
|
||||
|
|
@ -706,9 +705,17 @@ Response objects
|
|||
|
||||
A :class:`twisted.internet.ssl.Certificate` object representing
|
||||
the server's SSL certificate.
|
||||
|
||||
|
||||
Only populated for ``https`` responses, ``None`` otherwise.
|
||||
|
||||
.. attribute:: Response.ip_address
|
||||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
||||
This attribute is currently only populated by the HTTP 1.1 download
|
||||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. method:: Response.copy()
|
||||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
|
@ -724,18 +731,16 @@ Response objects
|
|||
Constructs an absolute url by combining the Response's :attr:`url` with
|
||||
a possible relative url.
|
||||
|
||||
This is a wrapper over `urlparse.urljoin`_, it's merely an alias for
|
||||
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
|
||||
making this call::
|
||||
|
||||
urlparse.urljoin(response.url, url)
|
||||
urllib.parse.urljoin(response.url, url)
|
||||
|
||||
.. automethod:: Response.follow
|
||||
|
||||
.. automethod:: Response.follow_all
|
||||
|
||||
|
||||
.. _urlparse.urljoin: https://docs.python.org/2/library/urlparse.html#urlparse.urljoin
|
||||
|
||||
.. _topics-request-response-ref-response-subclasses:
|
||||
|
||||
Response subclasses
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ achieve this, such as:
|
|||
drawback: it's slow.
|
||||
|
||||
* `lxml`_ is an XML parsing library (which also parses HTML) with a pythonic
|
||||
API based on `ElementTree`_. (lxml is not part of the Python standard
|
||||
API based on :mod:`~xml.etree.ElementTree`. (lxml is not part of the Python standard
|
||||
library.)
|
||||
|
||||
Scrapy comes with its own mechanism for extracting data. They're called
|
||||
|
|
@ -36,7 +36,6 @@ defines selectors to associate those styles with specific HTML elements.
|
|||
|
||||
.. _BeautifulSoup: https://www.crummy.com/software/BeautifulSoup/
|
||||
.. _lxml: https://lxml.de/
|
||||
.. _ElementTree: https://docs.python.org/2/library/xml.etree.elementtree.html
|
||||
.. _XPath: https://www.w3.org/TR/xpath/all/
|
||||
.. _CSS: https://www.w3.org/TR/selectors
|
||||
.. _parsel: https://parsel.readthedocs.io/en/latest/
|
||||
|
|
|
|||
|
|
@ -26,9 +26,7 @@ do this by using an environment variable, ``SCRAPY_SETTINGS_MODULE``.
|
|||
|
||||
The value of ``SCRAPY_SETTINGS_MODULE`` should be in Python path syntax, e.g.
|
||||
``myproject.settings``. Note that the settings module should be on the
|
||||
Python `import search path`_.
|
||||
|
||||
.. _import search path: https://docs.python.org/2/tutorial/modules.html#the-module-search-path
|
||||
Python :ref:`import search path <tut-searchpath>`.
|
||||
|
||||
.. _populating-settings:
|
||||
|
||||
|
|
@ -899,10 +897,9 @@ LOG_FORMAT
|
|||
|
||||
Default: ``'%(asctime)s [%(name)s] %(levelname)s: %(message)s'``
|
||||
|
||||
String for formatting log messages. Refer to the `Python logging documentation`_ for the whole list of available
|
||||
placeholders.
|
||||
|
||||
.. _Python logging documentation: https://docs.python.org/2/library/logging.html#logrecord-attributes
|
||||
String for formatting log messages. Refer to the
|
||||
:ref:`Python logging documentation <logrecord-attributes>` for the qwhole
|
||||
list of available placeholders.
|
||||
|
||||
.. setting:: LOG_DATEFORMAT
|
||||
|
||||
|
|
@ -912,10 +909,9 @@ LOG_DATEFORMAT
|
|||
Default: ``'%Y-%m-%d %H:%M:%S'``
|
||||
|
||||
String for formatting date/time, expansion of the ``%(asctime)s`` placeholder
|
||||
in :setting:`LOG_FORMAT`. Refer to the `Python datetime documentation`_ for the whole list of available
|
||||
directives.
|
||||
|
||||
.. _Python datetime documentation: https://docs.python.org/2/library/datetime.html#strftime-and-strptime-behavior
|
||||
in :setting:`LOG_FORMAT`. Refer to the
|
||||
:ref:`Python datetime documentation <strftime-strptime-behavior>` for the
|
||||
whole list of available directives.
|
||||
|
||||
.. setting:: LOG_FORMATTER
|
||||
|
||||
|
|
@ -1116,17 +1112,6 @@ multi-purpose thread pool used by various Scrapy components. Threaded
|
|||
DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
|
||||
this value if you're experiencing problems with insufficient blocking IO.
|
||||
|
||||
.. setting:: REDIRECT_MAX_TIMES
|
||||
|
||||
REDIRECT_MAX_TIMES
|
||||
------------------
|
||||
|
||||
Default: ``20``
|
||||
|
||||
Defines the maximum times a request can be redirected. After this maximum the
|
||||
request's response is returned as is. We used Firefox default value for the
|
||||
same task.
|
||||
|
||||
.. setting:: REDIRECT_PRIORITY_ADJUST
|
||||
|
||||
REDIRECT_PRIORITY_ADJUST
|
||||
|
|
@ -1422,17 +1407,6 @@ Default: ``True``
|
|||
A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
|
||||
will be enabled (provided its extension is also enabled).
|
||||
|
||||
.. setting:: TELNETCONSOLE_PORT
|
||||
|
||||
TELNETCONSOLE_PORT
|
||||
------------------
|
||||
|
||||
Default: ``[6023, 6073]``
|
||||
|
||||
The port range to use for the telnet console. If set to ``None`` or ``0``, a
|
||||
dynamically assigned port is used. For more info see
|
||||
:ref:`topics-telnetconsole`.
|
||||
|
||||
.. setting:: TEMPLATES_DIR
|
||||
|
||||
TEMPLATES_DIR
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type response: :class:`~scrapy.http.Response` object
|
||||
|
||||
:param exception: the exception raised
|
||||
:type exception: `Exception`_ object
|
||||
:type exception: :exc:`Exception` object
|
||||
|
||||
:param spider: the spider which raised the exception
|
||||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
|
@ -173,20 +173,16 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
|
|||
:type spider: :class:`~scrapy.spiders.Spider` object
|
||||
|
||||
.. method:: from_crawler(cls, crawler)
|
||||
|
||||
|
||||
If present, this classmethod is called to create a middleware instance
|
||||
from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
|
||||
of the middleware. Crawler object provides access to all Scrapy core
|
||||
components like settings and signals; it is a way for middleware to
|
||||
access them and hook its functionality into Scrapy.
|
||||
|
||||
|
||||
:param crawler: crawler that uses this middleware
|
||||
:type crawler: :class:`~scrapy.crawler.Crawler` object
|
||||
|
||||
|
||||
.. _Exception: https://docs.python.org/2/library/exceptions.html#exceptions.Exception
|
||||
|
||||
|
||||
.. _topics-spider-middleware-ref:
|
||||
|
||||
Built-in spider middleware reference
|
||||
|
|
|
|||
|
|
@ -298,9 +298,7 @@ Keep in mind that spider arguments are only strings.
|
|||
The spider will not do any parsing on its own.
|
||||
If you were to set the ``start_urls`` attribute from the command line,
|
||||
you would have to parse it on your own into a list
|
||||
using something like
|
||||
`ast.literal_eval <https://docs.python.org/3/library/ast.html#ast.literal_eval>`_
|
||||
or `json.loads <https://docs.python.org/3/library/json.html#json.loads>`_
|
||||
using something like :func:`ast.literal_eval` or :func:`json.loads`
|
||||
and then set it as an attribute.
|
||||
Otherwise, you would cause iteration over a ``start_urls`` string
|
||||
(a very common python pitfall)
|
||||
|
|
|
|||
|
|
@ -40,10 +40,10 @@ the console you need to type::
|
|||
Connected to localhost.
|
||||
Escape character is '^]'.
|
||||
Username:
|
||||
Password:
|
||||
Password:
|
||||
>>>
|
||||
|
||||
By default Username is ``scrapy`` and Password is autogenerated. The
|
||||
By default Username is ``scrapy`` and Password is autogenerated. The
|
||||
autogenerated Password can be seen on Scrapy logs like the example below::
|
||||
|
||||
2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
|
||||
|
|
@ -63,7 +63,7 @@ Available variables in the telnet console
|
|||
=========================================
|
||||
|
||||
The telnet console is like a regular Python shell running inside the Scrapy
|
||||
process, so you can do anything from it including importing new modules, etc.
|
||||
process, so you can do anything from it including importing new modules, etc.
|
||||
|
||||
However, the telnet console comes with some default variables defined for
|
||||
convenience:
|
||||
|
|
@ -89,13 +89,11 @@ convenience:
|
|||
+----------------+-------------------------------------------------------------------+
|
||||
| ``prefs`` | for memory debugging (see :ref:`topics-leaks`) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``p`` | a shortcut to the `pprint.pprint`_ function |
|
||||
| ``p`` | a shortcut to the :func:`pprint.pprint` function |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
| ``hpy`` | for memory debugging (see :ref:`topics-leaks`) |
|
||||
+----------------+-------------------------------------------------------------------+
|
||||
|
||||
.. _pprint.pprint: https://docs.python.org/library/pprint.html#pprint.pprint
|
||||
|
||||
Telnet console usage examples
|
||||
=============================
|
||||
|
||||
|
|
@ -208,4 +206,3 @@ Default: ``None``
|
|||
|
||||
The password used for the telnet console, default behaviour is to have it
|
||||
autogenerated
|
||||
|
||||
|
|
|
|||
22
pytest.ini
22
pytest.ini
|
|
@ -30,9 +30,9 @@ flake8-ignore =
|
|||
# Issues pending a review:
|
||||
# scrapy/commands
|
||||
scrapy/commands/__init__.py E128
|
||||
scrapy/commands/fetch.py E401 E128 E731
|
||||
scrapy/commands/fetch.py E401 E128
|
||||
scrapy/commands/genspider.py E128
|
||||
scrapy/commands/parse.py E128 E731
|
||||
scrapy/commands/parse.py E128
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128
|
||||
scrapy/commands/startproject.py E127 E128
|
||||
|
|
@ -43,9 +43,9 @@ flake8-ignore =
|
|||
# scrapy/core
|
||||
scrapy/core/engine.py E128 E127
|
||||
scrapy/core/scraper.py E128 W504
|
||||
scrapy/core/spidermw.py E731 E126
|
||||
scrapy/core/spidermw.py E126
|
||||
scrapy/core/downloader/contextfactory.py E128 E126
|
||||
scrapy/core/downloader/webclient.py E731 E128 E126
|
||||
scrapy/core/downloader/webclient.py E128 E126
|
||||
scrapy/core/downloader/handlers/ftp.py E128 E127
|
||||
scrapy/core/downloader/handlers/s3.py E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
|
|
@ -63,8 +63,7 @@ flake8-ignore =
|
|||
scrapy/http/response/__init__.py E128
|
||||
scrapy/http/response/text.py E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E402 W504
|
||||
scrapy/linkextractors/lxmlhtml.py E731
|
||||
scrapy/linkextractors/__init__.py E402 W504
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E128
|
||||
# scrapy/pipelines
|
||||
|
|
@ -114,7 +113,7 @@ flake8-ignore =
|
|||
tests/test_crawler.py F841
|
||||
tests/test_dependencies.py F841
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E126 E123
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E128 E126
|
||||
tests/test_downloadermiddleware_cookies.py E741 E128 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_httpcompression.py E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E128
|
||||
|
|
@ -122,20 +121,20 @@ flake8-ignore =
|
|||
tests/test_downloadermiddleware_retry.py E128 E126
|
||||
tests/test_dupefilters.py E741 E128 E124
|
||||
tests/test_engine.py E401 E128
|
||||
tests/test_exporters.py E731 E128 E124
|
||||
tests/test_exporters.py E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py F841
|
||||
tests/test_http_request.py E402 E127 E128 E128 E126 E123
|
||||
tests/test_http_response.py E128
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_linkextractors.py E128 E124
|
||||
tests/test_loader.py E731 E741 E128 E117
|
||||
tests/test_loader.py E741 E128 E117
|
||||
tests/test_logformatter.py E128 E122
|
||||
tests/test_mail.py E128
|
||||
tests/test_middleware.py E128
|
||||
tests/test_pipeline_crawl.py E128 E126
|
||||
tests/test_pipeline_images.py F841
|
||||
tests/test_pipeline_media.py E741 E731 E128
|
||||
tests/test_pipeline_media.py E741 E128
|
||||
tests/test_proxy_connect.py E741
|
||||
tests/test_scheduler.py E126 E123
|
||||
tests/test_selector.py E127
|
||||
|
|
@ -150,10 +149,9 @@ flake8-ignore =
|
|||
tests/test_utils_http.py E128 W504
|
||||
tests/test_utils_iterators.py E128 E129
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E731
|
||||
tests/test_utils_reqser.py E128
|
||||
tests/test_utils_request.py E128
|
||||
tests/test_utils_signal.py E741 F841 E731
|
||||
tests/test_utils_signal.py E741 F841
|
||||
tests/test_utils_sitemap.py E128 E124
|
||||
tests/test_utils_url.py E127 E125 E126 E123
|
||||
tests/test_webclient.py E128 E122 E402 E123 E126
|
||||
|
|
|
|||
|
|
@ -49,8 +49,8 @@ class Command(ScrapyCommand):
|
|||
def run(self, args, opts):
|
||||
if len(args) != 1 or not is_url(args[0]):
|
||||
raise UsageError()
|
||||
cb = lambda x: self._print_response(x, opts)
|
||||
request = Request(args[0], callback=cb, dont_filter=True)
|
||||
request = Request(args[0], callback=self._print_response,
|
||||
cb_kwargs={"opts": opts}, dont_filter=True)
|
||||
# by default, let the framework handle redirects,
|
||||
# i.e. command handles all codes expect 3xx
|
||||
if not opts.no_redirect:
|
||||
|
|
|
|||
|
|
@ -146,9 +146,8 @@ class Command(ScrapyCommand):
|
|||
if not self.spidercls:
|
||||
logger.error('Unable to find spider for: %(url)s', {'url': url})
|
||||
|
||||
# Request requires callback argument as callable or None, not string
|
||||
request = Request(url, None)
|
||||
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
|
||||
def _start_requests(spider):
|
||||
yield self.prepare_request(spider, Request(url), opts)
|
||||
self.spidercls.start_requests = _start_requests
|
||||
|
||||
def start_parsing(self, url, opts):
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ class Downloader:
|
|||
return response
|
||||
dfd.addCallback(_downloaded)
|
||||
|
||||
# 3. After response arrives, remove the request from transferring
|
||||
# 3. After response arrives, remove the request from transferring
|
||||
# state to free up the transferring slot so it can be used by the
|
||||
# following requests (perhaps those which came from the downloader
|
||||
# middleware itself)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Download handlers for http and https schemes"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
|
|
@ -373,7 +374,13 @@ class ScrapyAgent:
|
|||
def _cb_bodyready(self, txresponse, request):
|
||||
# deliverBody hangs for responses without body
|
||||
if txresponse.length == 0:
|
||||
return txresponse, b'', None, None
|
||||
return {
|
||||
"txresponse": txresponse,
|
||||
"body": b"",
|
||||
"flags": None,
|
||||
"certificate": None,
|
||||
"ip_address": None,
|
||||
}
|
||||
|
||||
maxsize = request.meta.get('download_maxsize', self._maxsize)
|
||||
warnsize = request.meta.get('download_warnsize', self._warnsize)
|
||||
|
|
@ -409,12 +416,17 @@ class ScrapyAgent:
|
|||
return d
|
||||
|
||||
def _cb_bodydone(self, result, request, url):
|
||||
txresponse, body, flags, certificate = result
|
||||
status = int(txresponse.code)
|
||||
headers = Headers(txresponse.headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
return respcls(url=url, status=status, headers=headers, body=body,
|
||||
flags=flags, certificate=certificate)
|
||||
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
|
||||
return respcls(
|
||||
url=url,
|
||||
status=int(result["txresponse"].code),
|
||||
headers=headers,
|
||||
body=result["body"],
|
||||
flags=result["flags"],
|
||||
certificate=result["certificate"],
|
||||
ip_address=result["ip_address"],
|
||||
)
|
||||
|
||||
|
||||
@implementer(IBodyProducer)
|
||||
|
|
@ -449,12 +461,16 @@ class _ResponseReader(protocol.Protocol):
|
|||
self._reached_warnsize = False
|
||||
self._bytes_received = 0
|
||||
self._certificate = None
|
||||
self._ip_address = None
|
||||
|
||||
def connectionMade(self):
|
||||
if self._certificate is None:
|
||||
with suppress(AttributeError):
|
||||
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
|
||||
|
||||
if self._ip_address is None:
|
||||
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
|
||||
|
||||
def dataReceived(self, bodyBytes):
|
||||
# This maybe called several times after cancel was called with buffered data.
|
||||
if self._finished.called:
|
||||
|
|
@ -486,16 +502,34 @@ class _ResponseReader(protocol.Protocol):
|
|||
|
||||
body = self._bodybuf.getvalue()
|
||||
if reason.check(ResponseDone):
|
||||
self._finished.callback((self._txresponse, body, None, self._certificate))
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": None,
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
if reason.check(PotentialDataLoss):
|
||||
self._finished.callback((self._txresponse, body, ['partial'], self._certificate))
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["partial"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
|
||||
if not self._fail_on_dataloss:
|
||||
self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate))
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["dataloss"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
elif not self._fail_on_dataloss_warned:
|
||||
|
|
|
|||
|
|
@ -40,14 +40,14 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
% (method.__self__.__class__.__name__, response.__class__.__name__)
|
||||
)
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue((yield download_func(request=request, spider=spider)))
|
||||
return response
|
||||
return (yield download_func(request=request, spider=spider))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def process_response(response):
|
||||
assert response is not None, 'Received None in process_response'
|
||||
if isinstance(response, Request):
|
||||
defer.returnValue(response)
|
||||
return response
|
||||
|
||||
for method in self.methods['process_response']:
|
||||
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
|
||||
|
|
@ -57,12 +57,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
% (method.__self__.__class__.__name__, type(response))
|
||||
)
|
||||
if isinstance(response, Request):
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(response)
|
||||
return response
|
||||
return response
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def process_exception(_failure):
|
||||
exception = _failure.value
|
||||
def process_exception(failure):
|
||||
exception = failure.value
|
||||
for method in self.methods['process_exception']:
|
||||
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
|
||||
if response is not None and not isinstance(response, (Response, Request)):
|
||||
|
|
@ -71,8 +71,8 @@ class DownloaderMiddlewareManager(MiddlewareManager):
|
|||
% (method.__self__.__class__.__name__, type(response))
|
||||
)
|
||||
if response:
|
||||
defer.returnValue(response)
|
||||
defer.returnValue(_failure)
|
||||
return response
|
||||
return failure
|
||||
|
||||
deferred = mustbe_deferred(process_request, request)
|
||||
deferred.addErrback(process_exception)
|
||||
|
|
|
|||
|
|
@ -14,13 +14,12 @@ from scrapy.responsetypes import responsetypes
|
|||
def _parsed_url_args(parsed):
|
||||
# Assume parsed is urlparse-d from Request.url,
|
||||
# which was passed via safe_url_string and is ascii-only.
|
||||
b = lambda s: to_bytes(s, encoding='ascii')
|
||||
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
|
||||
path = b(path)
|
||||
host = b(parsed.hostname)
|
||||
path = to_bytes(path, encoding="ascii")
|
||||
host = to_bytes(parsed.hostname, encoding="ascii")
|
||||
port = parsed.port
|
||||
scheme = b(parsed.scheme)
|
||||
netloc = b(parsed.netloc)
|
||||
scheme = to_bytes(parsed.scheme, encoding="ascii")
|
||||
netloc = to_bytes(parsed.netloc, encoding="ascii")
|
||||
if port is None:
|
||||
port = 443 if scheme == b'https' else 80
|
||||
return scheme, netloc, host, port, path
|
||||
|
|
|
|||
|
|
@ -129,6 +129,9 @@ class Request(object_ref):
|
|||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`,
|
||||
may modify the :class:`~scrapy.http.Request` object.
|
||||
|
||||
To translate a cURL command into a Scrapy request,
|
||||
you may use `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_.
|
||||
|
||||
"""
|
||||
request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options)
|
||||
request_kwargs.update(kwargs)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ from scrapy.utils.trackref import object_ref
|
|||
|
||||
class Response(object_ref):
|
||||
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None):
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None,
|
||||
request=None, certificate=None, ip_address=None):
|
||||
self.headers = Headers(headers or {})
|
||||
self.status = int(status)
|
||||
self._set_body(body)
|
||||
|
|
@ -25,6 +26,7 @@ class Response(object_ref):
|
|||
self.request = request
|
||||
self.flags = [] if flags is None else list(flags)
|
||||
self.certificate = certificate
|
||||
self.ip_address = ip_address
|
||||
|
||||
@property
|
||||
def cb_kwargs(self):
|
||||
|
|
@ -87,7 +89,8 @@ class Response(object_ref):
|
|||
"""Create a new Response with the same attributes except for those
|
||||
given new values.
|
||||
"""
|
||||
for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']:
|
||||
for x in ['url', 'status', 'headers', 'body',
|
||||
'request', 'flags', 'certificate', 'ip_address']:
|
||||
kwargs.setdefault(x, getattr(self, x))
|
||||
cls = kwargs.pop('cls', self.__class__)
|
||||
return cls(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -121,9 +121,7 @@ class DictItem(MutableMapping, BaseItem):
|
|||
return self.__class__(self)
|
||||
|
||||
def deepcopy(self):
|
||||
"""Return a `deep copy`_ of this item.
|
||||
|
||||
.. _deep copy: https://docs.python.org/library/copy.html#copy.deepcopy
|
||||
"""Return a :func:`~copy.deepcopy` of this item.
|
||||
"""
|
||||
return deepcopy(self)
|
||||
|
||||
|
|
|
|||
|
|
@ -45,8 +45,14 @@ IGNORED_EXTENSIONS = [
|
|||
|
||||
|
||||
_re_type = type(re.compile("", 0))
|
||||
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
|
||||
|
||||
|
||||
def _matches(url, regexs):
|
||||
return any(r.search(url) for r in regexs)
|
||||
|
||||
|
||||
def _is_valid_url(url):
|
||||
return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
|
||||
|
||||
|
||||
class FilteringLinkExtractor:
|
||||
|
|
|
|||
|
|
@ -98,11 +98,9 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
|
|||
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
|
||||
strip=True, restrict_text=None):
|
||||
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
lx = LxmlParserLinkExtractor(
|
||||
tag=tag_func,
|
||||
attr=attr_func,
|
||||
tag=lambda x: x in tags,
|
||||
attr=lambda x: x in attrs,
|
||||
unique=unique,
|
||||
process=process_value,
|
||||
strip=strip,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import functools
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return
|
||||
from twisted.internet.defer import Deferred, DeferredList
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -141,24 +141,26 @@ class MediaPipeline:
|
|||
# This code fixes a memory leak by avoiding to keep references to
|
||||
# the Request and Response objects on the Media Pipeline cache.
|
||||
#
|
||||
# Twisted inline callbacks pass return values using the function
|
||||
# twisted.internet.defer.returnValue, which encapsulates the return
|
||||
# value inside a _DefGen_Return base exception.
|
||||
#
|
||||
# What happens when the media_downloaded callback raises another
|
||||
# What happens when the media_downloaded callback raises an
|
||||
# exception, for example a FileException('download-error') when
|
||||
# the Response status code is not 200 OK, is that it stores the
|
||||
# _DefGen_Return exception on the FileException context.
|
||||
# the Response status code is not 200 OK, is that the original
|
||||
# StopIteration exception (which in turn contains the failed
|
||||
# Response and by extension, the original Request) gets encapsulated
|
||||
# within the FileException context.
|
||||
#
|
||||
# Originally, Scrapy was using twisted.internet.defer.returnValue
|
||||
# inside functions decorated with twisted.internet.defer.inlineCallbacks,
|
||||
# encapsulating the returned Response in a _DefGen_Return exception
|
||||
# instead of a StopIteration.
|
||||
#
|
||||
# To avoid keeping references to the Response and therefore Request
|
||||
# objects on the Media Pipeline cache, we should wipe the context of
|
||||
# the exception encapsulated by the Twisted Failure when its a
|
||||
# _DefGen_Return instance.
|
||||
# the encapsulated exception when it is a StopIteration instance
|
||||
#
|
||||
# This problem does not occur in Python 2.7 since we don't have
|
||||
# Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
|
||||
context = getattr(result.value, '__context__', None)
|
||||
if isinstance(context, _DefGen_Return):
|
||||
if isinstance(context, StopIteration):
|
||||
setattr(result.value, '__context__', None)
|
||||
|
||||
info.downloading.remove(fp)
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ class ResponseTypes:
|
|||
cls = Response
|
||||
if b'Content-Type' in headers:
|
||||
cls = self.from_content_type(
|
||||
content_type=headers[b'Content-type'],
|
||||
content_type=headers[b'Content-Type'],
|
||||
content_encoding=headers.get(b'Content-Encoding')
|
||||
)
|
||||
if cls is Response and b'Content-Disposition' in headers:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
from urllib.parse import urlparse
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.names.client import createResolver
|
||||
|
||||
from scrapy import Spider, Request
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
from tests.mockserver import MockServer, MockDNSServer
|
||||
|
||||
|
||||
class LocalhostSpider(Spider):
|
||||
name = "localhost_spider"
|
||||
|
||||
def start_requests(self):
|
||||
yield Request(self.url)
|
||||
|
||||
def parse(self, response):
|
||||
netloc = urlparse(response.url).netloc
|
||||
self.logger.info("Host: %s" % netloc.split(":")[0])
|
||||
self.logger.info("Type: %s" % type(response.ip_address))
|
||||
self.logger.info("IP address: %s" % response.ip_address)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
|
||||
port = urlparse(mock_http_server.http_address).port
|
||||
url = "http://not.a.real.domain:{port}/echo".format(port=port)
|
||||
|
||||
servers = [(mock_dns_server.host, mock_dns_server.port)]
|
||||
reactor.installResolver(createResolver(servers=servers))
|
||||
|
||||
configure_logging()
|
||||
runner = CrawlerRunner()
|
||||
d = runner.crawl(LocalhostSpider, url=url)
|
||||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run()
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
|
|
@ -6,18 +7,19 @@ from subprocess import Popen, PIPE
|
|||
from urllib.parse import urlencode
|
||||
|
||||
from OpenSSL import SSL
|
||||
from twisted.web.server import Site, NOT_DONE_YET
|
||||
from twisted.web.resource import Resource
|
||||
from twisted.internet import defer, reactor, ssl
|
||||
from twisted.internet.task import deferLater
|
||||
from twisted.names import dns, error
|
||||
from twisted.names.server import DNSServerFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper, Resource
|
||||
from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
|
||||
from twisted.web.static import File
|
||||
from twisted.web.test.test_webclient import PayloadResource
|
||||
from twisted.web.server import GzipEncoderFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper
|
||||
from twisted.web.util import redirectTo
|
||||
from twisted.internet import reactor, ssl
|
||||
from twisted.internet.task import deferLater
|
||||
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
|
||||
def getarg(request, name, default=None, type=None):
|
||||
|
|
@ -198,12 +200,10 @@ class Root(Resource):
|
|||
return b'Scrapy mock HTTP server\n'
|
||||
|
||||
|
||||
class MockServer():
|
||||
class MockServer:
|
||||
|
||||
def __enter__(self):
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'],
|
||||
stdout=PIPE, env=get_testenv())
|
||||
http_address = self.proc.stdout.readline().strip().decode('ascii')
|
||||
https_address = self.proc.stdout.readline().strip().decode('ascii')
|
||||
|
|
@ -224,11 +224,45 @@ class MockServer():
|
|||
return host + path
|
||||
|
||||
|
||||
class MockDNSResolver:
|
||||
"""
|
||||
Implements twisted.internet.interfaces.IResolver partially
|
||||
"""
|
||||
|
||||
def _resolve(self, name):
|
||||
record = dns.Record_A(address=b"127.0.0.1")
|
||||
answer = dns.RRHeader(name=name, payload=record)
|
||||
return [answer], [], []
|
||||
|
||||
def query(self, query, timeout=None):
|
||||
if query.type == dns.A:
|
||||
return defer.succeed(self._resolve(query.name.name))
|
||||
return defer.fail(error.DomainError())
|
||||
|
||||
def lookupAllRecords(self, name, timeout=None):
|
||||
return defer.succeed(self._resolve(name))
|
||||
|
||||
|
||||
class MockDNSServer:
|
||||
|
||||
def __enter__(self):
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
|
||||
stdout=PIPE, env=get_testenv())
|
||||
host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
|
||||
self.host = host
|
||||
self.port = int(port)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.proc.kill()
|
||||
self.proc.communicate()
|
||||
|
||||
|
||||
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
|
||||
factory = ssl.DefaultOpenSSLContextFactory(
|
||||
os.path.join(os.path.dirname(__file__), keyfile),
|
||||
os.path.join(os.path.dirname(__file__), certfile),
|
||||
)
|
||||
os.path.join(os.path.dirname(__file__), keyfile),
|
||||
os.path.join(os.path.dirname(__file__), certfile),
|
||||
)
|
||||
if cipher_string:
|
||||
ctx = factory.getContext()
|
||||
# disabling TLS1.2+ because it unconditionally enables some strong ciphers
|
||||
|
|
@ -238,19 +272,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
root = Root()
|
||||
factory = Site(root)
|
||||
httpPort = reactor.listenTCP(0, factory)
|
||||
contextFactory = ssl_context_factory()
|
||||
httpsPort = reactor.listenSSL(0, factory, contextFactory)
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http")
|
||||
args = parser.parse_args()
|
||||
|
||||
def print_listening():
|
||||
httpHost = httpPort.getHost()
|
||||
httpsHost = httpsPort.getHost()
|
||||
httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
|
||||
httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
|
||||
print(httpAddress)
|
||||
print(httpsAddress)
|
||||
if args.type == "http":
|
||||
root = Root()
|
||||
factory = Site(root)
|
||||
httpPort = reactor.listenTCP(0, factory)
|
||||
contextFactory = ssl_context_factory()
|
||||
httpsPort = reactor.listenSSL(0, factory, contextFactory)
|
||||
|
||||
def print_listening():
|
||||
httpHost = httpPort.getHost()
|
||||
httpsHost = httpsPort.getHost()
|
||||
httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port)
|
||||
httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port)
|
||||
print(httpAddress)
|
||||
print(httpsAddress)
|
||||
|
||||
elif args.type == "dns":
|
||||
clients = [MockDNSResolver()]
|
||||
factory = DNSServerFactory(clients=clients)
|
||||
protocol = dns.DNSDatagramProtocol(controller=factory)
|
||||
listener = reactor.listenUDP(0, protocol)
|
||||
|
||||
def print_listening():
|
||||
host = listener.getHost()
|
||||
print("%s:%s" % (host.host, host.port))
|
||||
|
||||
reactor.callWhenRunning(print_listening)
|
||||
reactor.run()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import json
|
||||
import logging
|
||||
import sys
|
||||
from ipaddress import IPv4Address
|
||||
from socket import gethostbyname
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pytest import mark
|
||||
from testfixtures import LogCapture
|
||||
|
|
@ -436,3 +439,21 @@ with multiples lines
|
|||
self.assertIsInstance(cert, Certificate)
|
||||
self.assertEqual(cert.getSubject().commonName, b"localhost")
|
||||
self.assertEqual(cert.getIssuer().commonName, b"localhost")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_dns_server_ip_address_none(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url('/status?n=200')
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
ip_address = crawler.spider.meta['responses'][0].ip_address
|
||||
self.assertIsNone(ip_address)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_dns_server_ip_address(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url('/echo?body=test')
|
||||
expected_netloc, _ = urlparse(url).netloc.split(':')
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
ip_address = crawler.spider.meta['responses'][0].ip_address
|
||||
self.assertIsInstance(ip_address, IPv4Address)
|
||||
self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
|
||||
|
|
|
|||
|
|
@ -274,9 +274,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
|
||||
|
||||
|
||||
class CrawlerProcessSubprocess(unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
|
||||
|
||||
class ScriptRunnerMixin:
|
||||
def run_script(self, script_name):
|
||||
script_path = os.path.join(self.script_dir, script_name)
|
||||
args = (sys.executable, script_path)
|
||||
|
|
@ -285,6 +283,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
|
|||
stdout, stderr = p.communicate()
|
||||
return stderr.decode('utf-8')
|
||||
|
||||
|
||||
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
|
||||
|
||||
def test_simple(self):
|
||||
log = self.run_script('simple.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
|
|
@ -334,3 +336,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
|
|||
log = self.run_script("twisted_reactor_asyncio.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
|
||||
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
|
||||
|
||||
def test_response_ip_address(self):
|
||||
log = self.run_script("ip_address.py")
|
||||
self.assertIn("INFO: Spider closed (finished)", log)
|
||||
self.assertIn("INFO: Host: not.a.real.domain", log)
|
||||
self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
|
||||
self.assertIn("INFO: IP address: 127.0.0.1", log)
|
||||
|
|
|
|||
|
|
@ -13,10 +13,9 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
|
|||
class CookiesMiddlewareTest(TestCase):
|
||||
|
||||
def assertCookieValEqual(self, first, second, msg=None):
|
||||
cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
|
||||
return self.assertEqual(
|
||||
sorted(cookievaleq(first)),
|
||||
sorted(cookievaleq(second)), msg)
|
||||
def split_cookies(cookies):
|
||||
return sorted(re.split(r";\s*", cookies.decode("latin1")))
|
||||
return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
|
||||
|
||||
def setUp(self):
|
||||
self.spider = Spider('foo')
|
||||
|
|
|
|||
|
|
@ -224,11 +224,12 @@ class CsvItemExporterTest(BaseItemExporterTest):
|
|||
return CsvItemExporter(self.output, **kwargs)
|
||||
|
||||
def assertCsvEqual(self, first, second, msg=None):
|
||||
first = to_unicode(first)
|
||||
second = to_unicode(second)
|
||||
csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
|
||||
for line in csv.splitlines(True)]
|
||||
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
|
||||
def split_csv(csv):
|
||||
return [
|
||||
sorted(re.split(r"(,|\s+)", line))
|
||||
for line in to_unicode(csv).splitlines(True)
|
||||
]
|
||||
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
|
||||
|
||||
def _check_output(self):
|
||||
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
|
||||
|
|
|
|||
|
|
@ -433,7 +433,7 @@ class FeedExportTest(unittest.TestCase):
|
|||
for file_path in FEEDS.keys():
|
||||
os.remove(str(file_path))
|
||||
|
||||
defer.returnValue(content)
|
||||
return content
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def exported_data(self, items, settings):
|
||||
|
|
@ -448,7 +448,7 @@ class FeedExportTest(unittest.TestCase):
|
|||
yield item
|
||||
|
||||
data = yield self.run_and_export(TestSpider, settings)
|
||||
defer.returnValue(data)
|
||||
return data
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def exported_no_data(self, settings):
|
||||
|
|
@ -462,7 +462,7 @@ class FeedExportTest(unittest.TestCase):
|
|||
pass
|
||||
|
||||
data = yield self.run_and_export(TestSpider, settings)
|
||||
defer.returnValue(data)
|
||||
return data
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from testfixtures import LogCapture
|
|||
from twisted.trial import unittest
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.internet import reactor
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
|
||||
from twisted.internet.defer import Deferred, inlineCallbacks
|
||||
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -124,9 +124,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
# Simulate the Media Pipeline behavior to produce a Twisted Failure
|
||||
try:
|
||||
# Simulate a Twisted inline callback returning a Response
|
||||
# The returnValue method raises an exception encapsulating the value
|
||||
returnValue(response)
|
||||
except BaseException as exc:
|
||||
raise StopIteration(response)
|
||||
except StopIteration as exc:
|
||||
def_gen_return_exc = exc
|
||||
try:
|
||||
# Simulate the media_downloaded callback raising a FileException
|
||||
|
|
@ -140,7 +139,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
|
||||
# The Failure should encapsulate a FileException ...
|
||||
self.assertEqual(failure.value, file_exc)
|
||||
# ... and it should have the returnValue exception set as its context
|
||||
# ... and it should have the StopIteration exception set as its context
|
||||
self.assertEqual(failure.value.__context__, def_gen_return_exc)
|
||||
|
||||
# Let's calculate the request fingerprint and fake some runtime data...
|
||||
|
|
@ -155,7 +154,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
|
|||
self.assertEqual(info.downloaded[fp], failure)
|
||||
# ... encapsulating the original FileException ...
|
||||
self.assertEqual(info.downloaded[fp].value, file_exc)
|
||||
# ... but it should not store the returnValue exception on its context
|
||||
# ... but it should not store the StopIteration exception on its context
|
||||
context = getattr(info.downloaded[fp].value, '__context__', None)
|
||||
self.assertIsNone(context)
|
||||
|
||||
|
|
@ -199,12 +198,19 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
|
||||
pipeline_class = MockedMediaPipeline
|
||||
|
||||
def _callback(self, result):
|
||||
self.pipe._mockcalled.append('request_callback')
|
||||
return result
|
||||
|
||||
def _errback(self, result):
|
||||
self.pipe._mockcalled.append('request_errback')
|
||||
return result
|
||||
|
||||
@inlineCallbacks
|
||||
def test_result_succeed(self):
|
||||
cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
|
||||
eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
|
||||
rsp = Response('http://url1')
|
||||
req = Request('http://url1', meta=dict(response=rsp), callback=cb, errback=eb)
|
||||
req = Request('http://url1', meta=dict(response=rsp),
|
||||
callback=self._callback, errback=self._errback)
|
||||
item = dict(requests=req)
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
self.assertEqual(new_item['results'], [(True, rsp)])
|
||||
|
|
@ -215,10 +221,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
@inlineCallbacks
|
||||
def test_result_failure(self):
|
||||
self.pipe.LOG_FAILED_RESULTS = False
|
||||
cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
|
||||
eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
|
||||
fail = Failure(Exception())
|
||||
req = Request('http://url1', meta=dict(response=fail), callback=cb, errback=eb)
|
||||
req = Request('http://url1', meta=dict(response=fail),
|
||||
callback=self._callback, errback=self._errback)
|
||||
item = dict(requests=req)
|
||||
new_item = yield self.pipe.process_item(item, self.spider)
|
||||
self.assertEqual(new_item['results'], [(False, fail)])
|
||||
|
|
|
|||
|
|
@ -292,7 +292,7 @@ class TestSpiderMiddleware(TestCase):
|
|||
crawler = get_crawler(spider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
raise defer.returnValue(log)
|
||||
return log
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_recovery(self):
|
||||
|
|
|
|||
|
|
@ -145,7 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
|
||||
get_z = operator.itemgetter('z')
|
||||
get_meta = operator.attrgetter('meta')
|
||||
compare_z = lambda obj: get_z(get_meta(obj))
|
||||
|
||||
def compare_z(obj):
|
||||
return get_z(get_meta(obj))
|
||||
|
||||
self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
|
||||
# fail z equality
|
||||
|
|
|
|||
|
|
@ -90,8 +90,10 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
|
|||
class SendCatchLogTest2(unittest.TestCase):
|
||||
|
||||
def test_error_logged_if_deferred_not_supported(self):
|
||||
def test_handler():
|
||||
return defer.Deferred()
|
||||
|
||||
test_signal = object()
|
||||
test_handler = lambda: defer.Deferred()
|
||||
dispatcher.connect(test_handler, test_signal)
|
||||
with LogCapture() as l:
|
||||
send_catch_log(test_signal)
|
||||
|
|
|
|||
6
tox.ini
6
tox.ini
|
|
@ -74,11 +74,15 @@ deps =
|
|||
changedir = docs
|
||||
deps =
|
||||
-rdocs/requirements.txt
|
||||
setenv =
|
||||
READTHEDOCS_PROJECT=scrapy
|
||||
READTHEDOCS_VERSION=master
|
||||
|
||||
[testenv:docs]
|
||||
basepython = python3
|
||||
changedir = {[docs]changedir}
|
||||
deps = {[docs]deps}
|
||||
setenv = {[docs]setenv}
|
||||
commands =
|
||||
sphinx-build -W -b html . {envtmpdir}/html
|
||||
|
||||
|
|
@ -86,6 +90,7 @@ commands =
|
|||
basepython = python3
|
||||
changedir = {[docs]changedir}
|
||||
deps = {[docs]deps}
|
||||
setenv = {[docs]setenv}
|
||||
commands =
|
||||
sphinx-build -b coverage . {envtmpdir}/coverage
|
||||
|
||||
|
|
@ -93,6 +98,7 @@ commands =
|
|||
basepython = python3
|
||||
changedir = {[docs]changedir}
|
||||
deps = {[docs]deps}
|
||||
setenv = {[docs]setenv}
|
||||
commands =
|
||||
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue