mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into response_ip_address
This commit is contained in:
Eugenio Lacuesta
commit
a44942d27d
|
|
@ -12,7 +12,10 @@ collect_ignore = [
|
|||
"scrapy/utils/testsite.py",
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
||||
*_py_files("tests/CrawlerProcess"),
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
|
||||
*_py_files("tests/CrawlerRunner"),
|
||||
# Py36-only parts of respective tests
|
||||
*_py_files("tests/py36"),
|
||||
]
|
||||
|
||||
for line in open('tests/ignores.txt'):
|
||||
|
|
|
|||
|
|
@ -81,35 +81,18 @@ Python packages can be installed either globally (a.k.a system wide),
|
|||
or in user-space. We do not recommend installing Scrapy system wide.
|
||||
|
||||
Instead, we recommend that you install Scrapy within a so-called
|
||||
"virtual environment" (`virtualenv`_).
|
||||
Virtualenvs allow you to not conflict with already-installed Python
|
||||
"virtual environment" (:mod:`venv`).
|
||||
Virtual environments allow you to not conflict with already-installed Python
|
||||
system packages (which could break some of your system tools and scripts),
|
||||
and still install packages normally with ``pip`` (without ``sudo`` and the likes).
|
||||
|
||||
To get started with virtual environments, see `virtualenv installation instructions`_.
|
||||
To install it globally (having it globally installed actually helps here),
|
||||
it should be a matter of running::
|
||||
See :ref:`tut-venv` on how to create your virtual environment.
|
||||
|
||||
$ [sudo] pip install virtualenv
|
||||
|
||||
Check this `user guide`_ on how to create your virtualenv.
|
||||
|
||||
.. note::
|
||||
If you use Linux or OS X, `virtualenvwrapper`_ is a handy tool to create virtualenvs.
|
||||
|
||||
Once you have created a virtualenv, you can install Scrapy inside it with ``pip``,
|
||||
Once you have created a virtual environment, you can install Scrapy inside it with ``pip``,
|
||||
just like any other Python package.
|
||||
(See :ref:`platform-specific guides <intro-install-platform-notes>`
|
||||
below for non-Python dependencies that you may need to install beforehand).
|
||||
|
||||
Python virtualenvs can be created to use Python 2 by default, or Python 3 by default. As Scrapy
|
||||
only supports Python 3, make sure you created a Python 3 virtualenv.
|
||||
|
||||
.. _virtualenv: https://virtualenv.pypa.io
|
||||
.. _virtualenv installation instructions: https://virtualenv.pypa.io/en/stable/installation/
|
||||
.. _virtualenvwrapper: https://virtualenvwrapper.readthedocs.io/en/latest/install.html
|
||||
.. _user guide: https://virtualenv.pypa.io/en/stable/userguide/
|
||||
|
||||
|
||||
.. _intro-install-platform-notes:
|
||||
|
||||
|
|
@ -205,15 +188,12 @@ solutions:
|
|||
|
||||
brew update; brew upgrade python
|
||||
|
||||
* *(Optional)* Install Scrapy inside an isolated python environment.
|
||||
* *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
|
||||
<intro-using-virtualenv>`.
|
||||
|
||||
This method is a workaround for the above OS X issue, but it's an overall
|
||||
good practice for managing dependencies and can complement the first method.
|
||||
|
||||
`virtualenv`_ is a tool you can use to create virtual environments in python.
|
||||
We recommended reading a tutorial like
|
||||
http://docs.python-guide.org/en/latest/dev/virtualenvs/ to get started.
|
||||
|
||||
After any of these workarounds you should be able to install Scrapy::
|
||||
|
||||
pip install Scrapy
|
||||
|
|
|
|||
|
|
@ -212,7 +212,7 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
|
|||
.. note::
|
||||
|
||||
Remember to always enclose urls in quotes when running Scrapy shell from
|
||||
command-line, otherwise urls containing arguments (ie. ``&`` character)
|
||||
command-line, otherwise urls containing arguments (i.e. ``&`` character)
|
||||
will not work.
|
||||
|
||||
On Windows, use double quotes instead::
|
||||
|
|
|
|||
|
|
@ -259,8 +259,8 @@ COOKIES_DEBUG
|
|||
|
||||
Default: ``False``
|
||||
|
||||
If enabled, Scrapy will log all cookies sent in requests (ie. ``Cookie``
|
||||
header) and all cookies received in responses (ie. ``Set-Cookie`` header).
|
||||
If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie``
|
||||
header) and all cookies received in responses (i.e. ``Set-Cookie`` header).
|
||||
|
||||
Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
|
||||
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ but disabled unless the :setting:`HTTPCACHE_ENABLED` setting is set.
|
|||
Disabling an extension
|
||||
======================
|
||||
|
||||
In order to disable an extension that comes enabled by default (ie. those
|
||||
In order to disable an extension that comes enabled by default (i.e. those
|
||||
included in the :setting:`EXTENSIONS_BASE` setting) you must set its order to
|
||||
``None``. For example::
|
||||
|
||||
|
|
@ -345,7 +345,7 @@ signal is received. The information dumped is the following:
|
|||
After the stack trace and engine status is dumped, the Scrapy process continues
|
||||
running normally.
|
||||
|
||||
This extension only works on POSIX-compliant platforms (ie. not Windows),
|
||||
This extension only works on POSIX-compliant platforms (i.e. not Windows),
|
||||
because the `SIGQUIT`_ and `SIGUSR2`_ signals are not available on Windows.
|
||||
|
||||
There are at least two ways to send Scrapy the `SIGQUIT`_ signal:
|
||||
|
|
@ -370,7 +370,7 @@ running normally.
|
|||
|
||||
For more info see `Debugging in Python`_.
|
||||
|
||||
This extension only works on POSIX-compliant platforms (ie. not Windows).
|
||||
This extension only works on POSIX-compliant platforms (i.e. not Windows).
|
||||
|
||||
.. _Python debugger: https://docs.python.org/2/library/pdb.html
|
||||
.. _Debugging in Python: https://pythonconquerstheuniverse.wordpress.com/2009/09/10/debugging-in-python/
|
||||
|
|
|
|||
|
|
@ -301,7 +301,7 @@ FEED_STORE_EMPTY
|
|||
|
||||
Default: ``False``
|
||||
|
||||
Whether to export empty feeds (ie. feeds with no items).
|
||||
Whether to export empty feeds (i.e. feeds with no items).
|
||||
|
||||
.. setting:: FEED_STORAGES
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ Job directory
|
|||
|
||||
To enable persistence support you just need to define a *job directory* through
|
||||
the ``JOBDIR`` setting. This directory will be for storing all required data to
|
||||
keep the state of a single job (ie. a spider run). It's important to note that
|
||||
keep the state of a single job (i.e. a spider run). It's important to note that
|
||||
this directory must not be shared by different spiders, or even different
|
||||
jobs/runs of the same spider, as it's meant to be used for storing the state of
|
||||
a *single* job.
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ LxmlLinkExtractor
|
|||
:type allow: a regular expression (or list of)
|
||||
|
||||
:param deny: a single regular expression (or list of regular expressions)
|
||||
that the (absolute) urls must match in order to be excluded (ie. not
|
||||
that the (absolute) urls must match in order to be excluded (i.e. not
|
||||
extracted). It has precedence over the ``allow`` parameter. If not
|
||||
given (or empty) it won't exclude any links.
|
||||
:type deny: a regular expression (or list of)
|
||||
|
|
|
|||
|
|
@ -609,7 +609,10 @@ Response objects
|
|||
|
||||
:param request: the initial value of the :attr:`Response.request` attribute.
|
||||
This represents the :class:`Request` that generated this response.
|
||||
:type request: :class:`Request` object
|
||||
:type request: scrapy.http.Request
|
||||
|
||||
:param certificate: an object representing the server's SSL certificate.
|
||||
:type certificate: twisted.internet.ssl.Certificate
|
||||
|
||||
:param ip_address: The IP address of the server from which the Response originated.
|
||||
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
|
||||
|
|
@ -667,7 +670,7 @@ Response objects
|
|||
.. attribute:: Response.meta
|
||||
|
||||
A shortcut to the :attr:`Request.meta` attribute of the
|
||||
:attr:`Response.request` object (ie. ``self.request.meta``).
|
||||
:attr:`Response.request` object (i.e. ``self.request.meta``).
|
||||
|
||||
Unlike the :attr:`Response.request` attribute, the :attr:`Response.meta`
|
||||
attribute is propagated along redirects and retries, so you will get
|
||||
|
|
@ -675,6 +678,18 @@ Response objects
|
|||
|
||||
.. seealso:: :attr:`Request.meta` attribute
|
||||
|
||||
.. attribute:: Response.cb_kwargs
|
||||
|
||||
A shortcut to the :attr:`Request.cb_kwargs` attribute of the
|
||||
:attr:`Response.request` object (i.e. ``self.request.cb_kwargs``).
|
||||
|
||||
Unlike the :attr:`Response.request` attribute, the
|
||||
:attr:`Response.cb_kwargs` attribute is propagated along redirects and
|
||||
retries, so you will get the original :attr:`Request.cb_kwargs` sent
|
||||
from your spider.
|
||||
|
||||
.. seealso:: :attr:`Request.cb_kwargs` attribute
|
||||
|
||||
.. attribute:: Response.flags
|
||||
|
||||
A list that contains flags for this response. Flags are labels used for
|
||||
|
|
@ -682,6 +697,13 @@ Response objects
|
|||
they're shown on the string representation of the Response (`__str__`
|
||||
method) which is used by the engine for logging.
|
||||
|
||||
.. attribute:: Response.certificate
|
||||
|
||||
A :class:`twisted.internet.ssl.Certificate` object representing
|
||||
the server's SSL certificate.
|
||||
|
||||
Only populated for ``https`` responses, ``None`` otherwise.
|
||||
|
||||
.. attribute:: Response.ip_address
|
||||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
|
@ -771,7 +793,7 @@ TextResponse objects
|
|||
1. the encoding passed in the ``__init__`` method ``encoding`` argument
|
||||
|
||||
2. the encoding declared in the Content-Type HTTP header. If this
|
||||
encoding is not valid (ie. unknown), it is ignored and the next
|
||||
encoding is not valid (i.e. unknown), it is ignored and the next
|
||||
resolution mechanism is tried.
|
||||
|
||||
3. the encoding declared in the response body. The TextResponse class
|
||||
|
|
|
|||
|
|
@ -986,7 +986,7 @@ a :class:`~scrapy.http.HtmlResponse` object like this::
|
|||
sel = Selector(html_response)
|
||||
|
||||
1. Select all ``<h1>`` elements from an HTML response body, returning a list of
|
||||
:class:`Selector` objects (ie. a :class:`SelectorList` object)::
|
||||
:class:`Selector` objects (i.e. a :class:`SelectorList` object)::
|
||||
|
||||
sel.xpath("//h1")
|
||||
|
||||
|
|
@ -1013,7 +1013,7 @@ instantiated with an :class:`~scrapy.http.XmlResponse` object::
|
|||
sel = Selector(xml_response)
|
||||
|
||||
1. Select all ``<product>`` elements from an XML response body, returning a list
|
||||
of :class:`Selector` objects (ie. a :class:`SelectorList` object)::
|
||||
of :class:`Selector` objects (i.e. a :class:`SelectorList` object)::
|
||||
|
||||
sel.xpath("//product")
|
||||
|
||||
|
|
|
|||
|
|
@ -248,7 +248,7 @@ CONCURRENT_REQUESTS
|
|||
|
||||
Default: ``16``
|
||||
|
||||
The maximum number of concurrent (ie. simultaneous) requests that will be
|
||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||
performed by the Scrapy downloader.
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
|
|
@ -258,7 +258,7 @@ CONCURRENT_REQUESTS_PER_DOMAIN
|
|||
|
||||
Default: ``8``
|
||||
|
||||
The maximum number of concurrent (ie. simultaneous) requests that will be
|
||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||
performed to any single domain.
|
||||
|
||||
See also: :ref:`topics-autothrottle` and its
|
||||
|
|
@ -272,7 +272,7 @@ CONCURRENT_REQUESTS_PER_IP
|
|||
|
||||
Default: ``0``
|
||||
|
||||
The maximum number of concurrent (ie. simultaneous) requests that will be
|
||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||
performed to any single IP. If non-zero, the
|
||||
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is
|
||||
used instead. In other words, concurrency limits will be applied per IP, not
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ item_error
|
|||
.. signal:: item_error
|
||||
.. function:: item_error(item, response, spider, failure)
|
||||
|
||||
Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises
|
||||
Sent when a :ref:`topics-item-pipeline` generates an error (i.e. raises
|
||||
an exception), except :exc:`~scrapy.exceptions.DropItem` exception.
|
||||
|
||||
This signal supports returning deferreds from their handlers.
|
||||
|
|
@ -232,7 +232,7 @@ spider_error
|
|||
.. signal:: spider_error
|
||||
.. function:: spider_error(failure, response, spider)
|
||||
|
||||
Sent when a spider callback generates an error (ie. raises an exception).
|
||||
Sent when a spider callback generates an error (i.e. raises an exception).
|
||||
|
||||
This signal does not support returning deferreds from their handlers.
|
||||
|
||||
|
|
|
|||
115
pytest.ini
115
pytest.ini
|
|
@ -37,7 +37,7 @@ flake8-ignore =
|
|||
scrapy/commands/edit.py E501
|
||||
scrapy/commands/fetch.py E401 E501 E128 E731
|
||||
scrapy/commands/genspider.py E128 E501 E502
|
||||
scrapy/commands/parse.py E128 E501 E731 E226
|
||||
scrapy/commands/parse.py E128 E501 E731
|
||||
scrapy/commands/runspider.py E501
|
||||
scrapy/commands/settings.py E128
|
||||
scrapy/commands/shell.py E128 E501 E502
|
||||
|
|
@ -47,22 +47,22 @@ flake8-ignore =
|
|||
scrapy/contracts/__init__.py E501 W504
|
||||
scrapy/contracts/default.py E128
|
||||
# scrapy/core
|
||||
scrapy/core/engine.py E501 E128 E127 E306 E502
|
||||
scrapy/core/engine.py E501 E128 E127 E502
|
||||
scrapy/core/scheduler.py E501
|
||||
scrapy/core/scraper.py E501 E306 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E126 E226
|
||||
scrapy/core/scraper.py E501 E128 W504
|
||||
scrapy/core/spidermw.py E501 E731 E126
|
||||
scrapy/core/downloader/__init__.py E501
|
||||
scrapy/core/downloader/contextfactory.py E501 E128 E126
|
||||
scrapy/core/downloader/middleware.py E501 E502
|
||||
scrapy/core/downloader/tls.py E501 E305 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
|
||||
scrapy/core/downloader/tls.py E501 E241
|
||||
scrapy/core/downloader/webclient.py E731 E501 E128 E126
|
||||
scrapy/core/downloader/handlers/__init__.py E501
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
|
||||
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
|
||||
scrapy/core/downloader/handlers/http10.py E501
|
||||
scrapy/core/downloader/handlers/http11.py E501
|
||||
scrapy/core/downloader/handlers/s3.py E501 E128 E126
|
||||
# scrapy/downloadermiddlewares
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
|
||||
scrapy/downloadermiddlewares/ajaxcrawl.py E501
|
||||
scrapy/downloadermiddlewares/decompression.py E501
|
||||
scrapy/downloadermiddlewares/defaultheaders.py E501
|
||||
scrapy/downloadermiddlewares/httpcache.py E501 E126
|
||||
|
|
@ -76,7 +76,7 @@ flake8-ignore =
|
|||
scrapy/extensions/closespider.py E501 E128 E123
|
||||
scrapy/extensions/corestats.py E501
|
||||
scrapy/extensions/feedexport.py E128 E501
|
||||
scrapy/extensions/httpcache.py E128 E501 E303
|
||||
scrapy/extensions/httpcache.py E128 E501
|
||||
scrapy/extensions/memdebug.py E501
|
||||
scrapy/extensions/spiderstate.py E501
|
||||
scrapy/extensions/telnet.py E501 W504
|
||||
|
|
@ -91,7 +91,7 @@ flake8-ignore =
|
|||
scrapy/http/response/text.py E501 E128 E124
|
||||
# scrapy/linkextractors
|
||||
scrapy/linkextractors/__init__.py E731 E501 E402 W504
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
|
||||
scrapy/linkextractors/lxmlhtml.py E501 E731
|
||||
# scrapy/loader
|
||||
scrapy/loader/__init__.py E501 E128
|
||||
scrapy/loader/processors.py E501
|
||||
|
|
@ -105,7 +105,7 @@ flake8-ignore =
|
|||
scrapy/selector/unified.py E501 E111
|
||||
# scrapy/settings
|
||||
scrapy/settings/__init__.py E501
|
||||
scrapy/settings/default_settings.py E501 E114 E116 E226
|
||||
scrapy/settings/default_settings.py E501 E114 E116
|
||||
scrapy/settings/deprecated.py E501
|
||||
# scrapy/spidermiddlewares
|
||||
scrapy/spidermiddlewares/httperror.py E501
|
||||
|
|
@ -121,28 +121,27 @@ flake8-ignore =
|
|||
scrapy/utils/asyncio.py E501
|
||||
scrapy/utils/benchserver.py E501
|
||||
scrapy/utils/conf.py E402 E501
|
||||
scrapy/utils/console.py E306 E305
|
||||
scrapy/utils/datatypes.py E501 E226
|
||||
scrapy/utils/datatypes.py E501
|
||||
scrapy/utils/decorators.py E501
|
||||
scrapy/utils/defer.py E501 E128
|
||||
scrapy/utils/deprecate.py E128 E501 E127 E502
|
||||
scrapy/utils/gz.py E305 E501 W504
|
||||
scrapy/utils/http.py F403 E226
|
||||
scrapy/utils/gz.py E501 W504
|
||||
scrapy/utils/http.py F403
|
||||
scrapy/utils/httpobj.py E501
|
||||
scrapy/utils/iterators.py E501 E701
|
||||
scrapy/utils/iterators.py E501
|
||||
scrapy/utils/log.py E128 E501
|
||||
scrapy/utils/markup.py F403
|
||||
scrapy/utils/misc.py E501 E226
|
||||
scrapy/utils/misc.py E501
|
||||
scrapy/utils/multipart.py F403
|
||||
scrapy/utils/project.py E501
|
||||
scrapy/utils/python.py E501
|
||||
scrapy/utils/reactor.py E226 E501
|
||||
scrapy/utils/reactor.py E501
|
||||
scrapy/utils/reqser.py E501
|
||||
scrapy/utils/request.py E127 E501
|
||||
scrapy/utils/response.py E501 E128
|
||||
scrapy/utils/signal.py E501 E128
|
||||
scrapy/utils/sitemap.py E501
|
||||
scrapy/utils/spider.py E271 E501
|
||||
scrapy/utils/spider.py E501
|
||||
scrapy/utils/ssl.py E501
|
||||
scrapy/utils/test.py E501
|
||||
scrapy/utils/url.py E501 F403 E128 F405
|
||||
|
|
@ -152,7 +151,7 @@ flake8-ignore =
|
|||
scrapy/crawler.py E501
|
||||
scrapy/dupefilters.py E501 E202
|
||||
scrapy/exceptions.py E501
|
||||
scrapy/exporters.py E501 E226
|
||||
scrapy/exporters.py E501
|
||||
scrapy/interfaces.py E501
|
||||
scrapy/item.py E501 E128
|
||||
scrapy/link.py E501
|
||||
|
|
@ -161,93 +160,91 @@ flake8-ignore =
|
|||
scrapy/middleware.py E128 E501
|
||||
scrapy/pqueues.py E501
|
||||
scrapy/resolver.py E501
|
||||
scrapy/responsetypes.py E128 E501 E305
|
||||
scrapy/responsetypes.py E128 E501
|
||||
scrapy/robotstxt.py E501
|
||||
scrapy/shell.py E501
|
||||
scrapy/signalmanager.py E501
|
||||
scrapy/spiderloader.py E225 F841 E501 E126
|
||||
scrapy/spiderloader.py F841 E501 E126
|
||||
scrapy/squeues.py E128
|
||||
scrapy/statscollectors.py E501
|
||||
# tests
|
||||
tests/__init__.py E402 E501
|
||||
tests/mockserver.py E401 E501 E126 E123
|
||||
tests/pipelines.py F841 E226
|
||||
tests/pipelines.py F841
|
||||
tests/spiders.py E501 E127
|
||||
tests/test_closespider.py E501 E127
|
||||
tests/test_command_fetch.py E501
|
||||
tests/test_command_parse.py E501 E128 E303 E226
|
||||
tests/test_command_parse.py E501 E128
|
||||
tests/test_command_shell.py E501 E128
|
||||
tests/test_commands.py E128 E501
|
||||
tests/test_contracts.py E501 E128
|
||||
tests/test_crawl.py E501 E741 E265
|
||||
tests/test_crawler.py F841 E306 E501
|
||||
tests/test_dependencies.py F841 E501 E305
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
|
||||
tests/test_crawler.py F841 E501
|
||||
tests/test_dependencies.py F841 E501
|
||||
tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
|
||||
tests/test_downloadermiddleware.py E501
|
||||
tests/test_downloadermiddleware_ajaxcrawlable.py E501
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
|
||||
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
|
||||
tests/test_downloadermiddleware_decompression.py E127
|
||||
tests/test_downloadermiddleware_defaultheaders.py E501
|
||||
tests/test_downloadermiddleware_downloadtimeout.py E501
|
||||
tests/test_downloadermiddleware_httpcache.py E501 E305
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
|
||||
tests/test_downloadermiddleware_httpcache.py E501
|
||||
tests/test_downloadermiddleware_httpcompression.py E501 E126 E123
|
||||
tests/test_downloadermiddleware_httpproxy.py E501 E128
|
||||
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
|
||||
tests/test_downloadermiddleware_redirect.py E501 E128 E127
|
||||
tests/test_downloadermiddleware_retry.py E501 E128 E126
|
||||
tests/test_downloadermiddleware_robotstxt.py E501
|
||||
tests/test_downloadermiddleware_stats.py E501
|
||||
tests/test_dupefilters.py E221 E501 E741 E128 E124
|
||||
tests/test_dupefilters.py E501 E741 E128 E124
|
||||
tests/test_engine.py E401 E501 E128
|
||||
tests/test_exporters.py E501 E731 E306 E128 E124
|
||||
tests/test_exporters.py E501 E731 E128 E124
|
||||
tests/test_extension_telnet.py F841
|
||||
tests/test_feedexport.py E501 F841 E241
|
||||
tests/test_http_cookies.py E501
|
||||
tests/test_http_headers.py E501
|
||||
tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
|
||||
tests/test_http_response.py E501 E301 E128 E265
|
||||
tests/test_item.py E701 E128 F841 E306
|
||||
tests/test_http_response.py E501 E128 E265
|
||||
tests/test_item.py E128 F841
|
||||
tests/test_link.py E501
|
||||
tests/test_linkextractors.py E501 E128 E124
|
||||
tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
|
||||
tests/test_loader.py E501 E731 E741 E128 E117 E241
|
||||
tests/test_logformatter.py E128 E501 E122
|
||||
tests/test_mail.py E128 E501 E305
|
||||
tests/test_mail.py E128 E501
|
||||
tests/test_middleware.py E501 E128
|
||||
tests/test_pipeline_crawl.py E131 E501 E128 E126
|
||||
tests/test_pipeline_files.py E501 E303 E272 E226
|
||||
tests/test_pipeline_images.py F841 E501 E303
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
|
||||
tests/test_pipeline_crawl.py E501 E128 E126
|
||||
tests/test_pipeline_files.py E501
|
||||
tests/test_pipeline_images.py F841 E501
|
||||
tests/test_pipeline_media.py E501 E741 E731 E128 E502
|
||||
tests/test_proxy_connect.py E501 E741
|
||||
tests/test_request_cb_kwargs.py E501
|
||||
tests/test_responsetypes.py E501 E305
|
||||
tests/test_responsetypes.py E501
|
||||
tests/test_robotstxt_interface.py E501 E501
|
||||
tests/test_scheduler.py E501 E126 E123
|
||||
tests/test_selector.py E501 E127
|
||||
tests/test_spider.py E501
|
||||
tests/test_spidermiddleware.py E501 E226
|
||||
tests/test_spidermiddleware.py E501
|
||||
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
|
||||
tests/test_spidermiddleware_offsite.py E501 E128 E111
|
||||
tests/test_spidermiddleware_output_chain.py E501 E226
|
||||
tests/test_spidermiddleware_output_chain.py E501
|
||||
tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
|
||||
tests/test_squeues.py E501 E701 E741
|
||||
tests/test_squeues.py E501 E741
|
||||
tests/test_utils_asyncio.py E501
|
||||
tests/test_utils_conf.py E501 E303 E128
|
||||
tests/test_utils_conf.py E501 E128
|
||||
tests/test_utils_curl.py E501
|
||||
tests/test_utils_datatypes.py E402 E501 E305
|
||||
tests/test_utils_defer.py E306 E501 F841 E226
|
||||
tests/test_utils_deprecate.py F841 E306 E501
|
||||
tests/test_utils_datatypes.py E402 E501
|
||||
tests/test_utils_defer.py E501 F841
|
||||
tests/test_utils_deprecate.py F841 E501
|
||||
tests/test_utils_http.py E501 E128 W504
|
||||
tests/test_utils_iterators.py E501 E128 E129 E303 E241
|
||||
tests/test_utils_log.py E741 E226
|
||||
tests/test_utils_python.py E501 E303 E731 E701 E305
|
||||
tests/test_utils_iterators.py E501 E128 E129 E241
|
||||
tests/test_utils_log.py E741
|
||||
tests/test_utils_python.py E501 E731
|
||||
tests/test_utils_reqser.py E501 E128
|
||||
tests/test_utils_request.py E501 E128 E305
|
||||
tests/test_utils_request.py E501 E128
|
||||
tests/test_utils_response.py E501
|
||||
tests/test_utils_signal.py E741 F841 E731 E226
|
||||
tests/test_utils_signal.py E741 F841 E731
|
||||
tests/test_utils_sitemap.py E128 E501 E124
|
||||
tests/test_utils_spider.py E305
|
||||
tests/test_utils_template.py E305
|
||||
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
|
||||
tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
|
||||
tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
|
||||
tests/test_cmdline/__init__.py E501
|
||||
tests/test_settings/__init__.py E501 E128
|
||||
tests/test_spiderloader/__init__.py E128 E501
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ class Command(ScrapyCommand):
|
|||
else:
|
||||
items = self.items.get(lvl, [])
|
||||
|
||||
print("# Scraped Items ", "-"*60)
|
||||
print("# Scraped Items ", "-" * 60)
|
||||
display.pprint([dict(x) for x in items], colorize=colour)
|
||||
|
||||
def print_requests(self, lvl=None, colour=True):
|
||||
|
|
@ -92,14 +92,14 @@ class Command(ScrapyCommand):
|
|||
else:
|
||||
requests = self.requests.get(lvl, [])
|
||||
|
||||
print("# Requests ", "-"*65)
|
||||
print("# Requests ", "-" * 65)
|
||||
display.pprint(requests, colorize=colour)
|
||||
|
||||
def print_results(self, opts):
|
||||
colour = not opts.nocolour
|
||||
|
||||
if opts.verbose:
|
||||
for level in range(1, self.max_level+1):
|
||||
for level in range(1, self.max_level + 1):
|
||||
print('\n>>> DEPTH LEVEL: %s <<<' % level)
|
||||
if not opts.noitems:
|
||||
self.print_items(level, colour)
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@
|
|||
import logging
|
||||
import re
|
||||
import warnings
|
||||
from contextlib import suppress
|
||||
from io import BytesIO
|
||||
from ipaddress import ip_address
|
||||
from time import time
|
||||
from urllib.parse import urldefrag
|
||||
|
||||
from twisted.internet import defer, protocol, reactor
|
||||
from twisted.internet import defer, protocol, reactor, ssl
|
||||
from twisted.internet.endpoints import TCP4ClientEndpoint
|
||||
from twisted.internet.error import TimeoutError
|
||||
from twisted.web.client import Agent, HTTPConnectionPool, ResponseDone, ResponseFailed, URI
|
||||
|
|
@ -419,11 +420,12 @@ class ScrapyAgent(object):
|
|||
return d
|
||||
|
||||
def _cb_bodydone(self, result, request, url):
|
||||
txresponse, body, flags, ip_address = result
|
||||
txresponse, body, flags, certificate, ip_address = result
|
||||
status = int(txresponse.code)
|
||||
headers = Headers(txresponse.headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address)
|
||||
return respcls(url=url, status=status, headers=headers, body=body,
|
||||
flags=flags, certificate=certificate, ip_address=ip_address)
|
||||
|
||||
|
||||
@implementer(IBodyProducer)
|
||||
|
|
@ -457,9 +459,14 @@ class _ResponseReader(protocol.Protocol):
|
|||
self._fail_on_dataloss_warned = False
|
||||
self._reached_warnsize = False
|
||||
self._bytes_received = 0
|
||||
self._certificate = None
|
||||
self._ip_address = None
|
||||
|
||||
def connectionMade(self):
|
||||
if self._certificate is None:
|
||||
with suppress(AttributeError):
|
||||
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
|
||||
|
||||
if self._ip_address is None:
|
||||
self._ip_address = ip_address(self.transport._producer.getPeer().host)
|
||||
|
||||
|
|
@ -494,16 +501,22 @@ class _ResponseReader(protocol.Protocol):
|
|||
|
||||
body = self._bodybuf.getvalue()
|
||||
if reason.check(ResponseDone):
|
||||
self._finished.callback((self._txresponse, body, None, self._ip_address))
|
||||
self._finished.callback(
|
||||
(self._txresponse, body, None, self._certificate, self._ip_address)
|
||||
)
|
||||
return
|
||||
|
||||
if reason.check(PotentialDataLoss):
|
||||
self._finished.callback((self._txresponse, body, ['partial'], self._ip_address))
|
||||
self._finished.callback(
|
||||
(self._txresponse, body, ['partial'], self._certificate, self._ip_address)
|
||||
)
|
||||
return
|
||||
|
||||
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
|
||||
if not self._fail_on_dataloss:
|
||||
self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address))
|
||||
self._finished.callback(
|
||||
(self._txresponse, body, ['dataloss'], self._certificate, self._ip_address)
|
||||
)
|
||||
return
|
||||
|
||||
elif not self._fail_on_dataloss_warned:
|
||||
|
|
|
|||
|
|
@ -89,4 +89,5 @@ class ScrapyClientTLSOptions(ClientTLSOptions):
|
|||
'from host "{}" (exception: {})'.format(
|
||||
self._hostnameASCII, repr(e)))
|
||||
|
||||
|
||||
DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
|
||||
|
|
|
|||
|
|
@ -140,7 +140,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
self.headers['Content-Length'] = 0
|
||||
|
||||
def _build_response(self, body, request):
|
||||
request.meta['download_latency'] = self.headers_time-self.start_time
|
||||
request.meta['download_latency'] = self.headers_time - self.start_time
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self._url)
|
||||
|
|
|
|||
|
|
@ -230,6 +230,7 @@ class ExecutionEngine(object):
|
|||
def _download(self, request, spider):
|
||||
slot = self.slot
|
||||
slot.add_request(request)
|
||||
|
||||
def _on_success(response):
|
||||
assert isinstance(response, (Response, Request))
|
||||
if isinstance(response, Response):
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ class Scheduler(object):
|
|||
if self.dqs is None:
|
||||
return
|
||||
try:
|
||||
self.dqs.push(request, -request.priority)
|
||||
self.dqs.push(request)
|
||||
except ValueError as e: # non serializable request
|
||||
if self.logunser:
|
||||
msg = ("Unable to serialize request: %(request)s - reason:"
|
||||
|
|
@ -135,35 +135,29 @@ class Scheduler(object):
|
|||
return True
|
||||
|
||||
def _mqpush(self, request):
|
||||
self.mqs.push(request, -request.priority)
|
||||
self.mqs.push(request)
|
||||
|
||||
def _dqpop(self):
|
||||
if self.dqs:
|
||||
return self.dqs.pop()
|
||||
|
||||
def _newmq(self, priority):
|
||||
""" Factory for creating memory queues. """
|
||||
return self.mqclass()
|
||||
|
||||
def _newdq(self, priority):
|
||||
""" Factory for creating disk queues. """
|
||||
path = join(self.dqdir, 'p%s' % (priority, ))
|
||||
return self.dqclass(path)
|
||||
|
||||
def _mq(self):
|
||||
""" Create a new priority queue instance, with in-memory storage """
|
||||
return create_instance(self.pqclass, None, self.crawler, self._newmq,
|
||||
serialize=False)
|
||||
return create_instance(self.pqclass,
|
||||
settings=None,
|
||||
crawler=self.crawler,
|
||||
downstream_queue_cls=self.mqclass,
|
||||
key='')
|
||||
|
||||
def _dq(self):
|
||||
""" Create a new priority queue instance, with disk storage """
|
||||
state = self._read_dqs_state(self.dqdir)
|
||||
q = create_instance(self.pqclass,
|
||||
None,
|
||||
self.crawler,
|
||||
self._newdq,
|
||||
state,
|
||||
serialize=True)
|
||||
settings=None,
|
||||
crawler=self.crawler,
|
||||
downstream_queue_cls=self.dqclass,
|
||||
key=self.dqdir,
|
||||
startprios=state)
|
||||
if q:
|
||||
logger.info("Resuming crawl (%(queuesize)d requests scheduled)",
|
||||
{'queuesize': len(q)}, extra={'spider': self.spider})
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ from scrapy import signals
|
|||
from scrapy.http import Request, Response
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.core.spidermw import SpiderMiddlewareManager
|
||||
from scrapy.utils.request import referer_str
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -158,9 +157,9 @@ class Scraper(object):
|
|||
if isinstance(exc, CloseSpider):
|
||||
self.crawler.engine.close_spider(spider, exc.reason or 'cancelled')
|
||||
return
|
||||
logger.error(
|
||||
"Spider error processing %(request)s (referer: %(referer)s)",
|
||||
{'request': request, 'referer': referer_str(request)},
|
||||
logkws = self.logformatter.spider_error(_failure, request, response, spider)
|
||||
logger.log(
|
||||
*logformatter_adapter(logkws),
|
||||
exc_info=failure_to_exc_info(_failure),
|
||||
extra={'spider': spider}
|
||||
)
|
||||
|
|
@ -208,16 +207,21 @@ class Scraper(object):
|
|||
"""
|
||||
if isinstance(download_failure, Failure) and not download_failure.check(IgnoreRequest):
|
||||
if download_failure.frames:
|
||||
logger.error('Error downloading %(request)s',
|
||||
{'request': request},
|
||||
exc_info=failure_to_exc_info(download_failure),
|
||||
extra={'spider': spider})
|
||||
logkws = self.logformatter.download_error(download_failure, request, spider)
|
||||
logger.log(
|
||||
*logformatter_adapter(logkws),
|
||||
extra={'spider': spider},
|
||||
exc_info=failure_to_exc_info(download_failure),
|
||||
)
|
||||
else:
|
||||
errmsg = download_failure.getErrorMessage()
|
||||
if errmsg:
|
||||
logger.error('Error downloading %(request)s: %(errmsg)s',
|
||||
{'request': request, 'errmsg': errmsg},
|
||||
extra={'spider': spider})
|
||||
logkws = self.logformatter.download_error(
|
||||
download_failure, request, spider, errmsg)
|
||||
logger.log(
|
||||
*logformatter_adapter(logkws),
|
||||
extra={'spider': spider},
|
||||
)
|
||||
|
||||
if spider_failure is not download_failure:
|
||||
return spider_failure
|
||||
|
|
@ -236,7 +240,7 @@ class Scraper(object):
|
|||
signal=signals.item_dropped, item=item, response=response,
|
||||
spider=spider, exception=output.value)
|
||||
else:
|
||||
logkws = self.logformatter.error(item, ex, response, spider)
|
||||
logkws = self.logformatter.item_error(item, ex, response, spider)
|
||||
logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
|
||||
exc_info=failure_to_exc_info(output))
|
||||
return self.signals.send_catch_log_deferred(
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
if _isiterable(result):
|
||||
# stop exception handling by handing control over to the
|
||||
# process_spider_output chain if an iterable has been returned
|
||||
return process_spider_output(result, method_index+1)
|
||||
return process_spider_output(result, method_index + 1)
|
||||
elif result is None:
|
||||
continue
|
||||
else:
|
||||
|
|
@ -103,12 +103,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
|
|||
# might fail directly if the output value is not a generator
|
||||
result = method(response=response, result=result, spider=spider)
|
||||
except Exception as ex:
|
||||
exception_result = process_spider_exception(Failure(ex), method_index+1)
|
||||
exception_result = process_spider_exception(Failure(ex), method_index + 1)
|
||||
if isinstance(exception_result, Failure):
|
||||
raise
|
||||
return exception_result
|
||||
if _isiterable(result):
|
||||
result = _evaluate_iterable(result, method_index+1, recovered)
|
||||
result = _evaluate_iterable(result, method_index + 1, recovered)
|
||||
else:
|
||||
msg = "Middleware {} must return an iterable, got {}"
|
||||
raise _InvalidOutput(msg.format(_fname(method), type(result)))
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ class AjaxCrawlMiddleware(object):
|
|||
return response
|
||||
|
||||
# scrapy already handles #! links properly
|
||||
ajax_crawl_request = request.replace(url=request.url+'#!')
|
||||
ajax_crawl_request = request.replace(url=request.url + '#!')
|
||||
logger.debug("Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
|
||||
{'ajax_crawl_request': ajax_crawl_request, 'request': request},
|
||||
extra={'spider': spider})
|
||||
|
|
|
|||
|
|
@ -173,12 +173,12 @@ class XmlItemExporter(BaseItemExporter):
|
|||
if hasattr(serialized_value, 'items'):
|
||||
self._beautify_newline()
|
||||
for subname, value in serialized_value.items():
|
||||
self._export_xml_field(subname, value, depth=depth+1)
|
||||
self._export_xml_field(subname, value, depth=depth + 1)
|
||||
self._beautify_indent(depth=depth)
|
||||
elif is_listlike(serialized_value):
|
||||
self._beautify_newline()
|
||||
for value in serialized_value:
|
||||
self._export_xml_field('value', value, depth=depth+1)
|
||||
self._export_xml_field('value', value, depth=depth + 1)
|
||||
self._beautify_indent(depth=depth)
|
||||
elif isinstance(serialized_value, str):
|
||||
self.xg.characters(serialized_value)
|
||||
|
|
|
|||
|
|
@ -17,15 +17,27 @@ from scrapy.utils.trackref import object_ref
|
|||
|
||||
class Response(object_ref):
|
||||
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None):
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None,
|
||||
request=None, certificate=None, ip_address=None):
|
||||
self.headers = Headers(headers or {})
|
||||
self.status = int(status)
|
||||
self._set_body(body)
|
||||
self._set_url(url)
|
||||
self.request = request
|
||||
self.flags = [] if flags is None else list(flags)
|
||||
self.certificate = certificate
|
||||
self.ip_address = ip_address
|
||||
|
||||
@property
|
||||
def cb_kwargs(self):
|
||||
try:
|
||||
return self.request.cb_kwargs
|
||||
except AttributeError:
|
||||
raise AttributeError(
|
||||
"Response.cb_kwargs not available, this response "
|
||||
"is not tied to any request"
|
||||
)
|
||||
|
||||
@property
|
||||
def meta(self):
|
||||
try:
|
||||
|
|
@ -77,7 +89,8 @@ class Response(object_ref):
|
|||
"""Create a new Response with the same attributes except for those
|
||||
given new values.
|
||||
"""
|
||||
for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']:
|
||||
for x in ['url', 'status', 'headers', 'body',
|
||||
'request', 'flags', 'certificate', 'ip_address']:
|
||||
kwargs.setdefault(x, getattr(self, x))
|
||||
cls = kwargs.pop('cls', self.__class__)
|
||||
return cls(*args, **kwargs)
|
||||
|
|
@ -108,7 +121,7 @@ class Response(object_ref):
|
|||
|
||||
def follow(self, url, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding='utf-8', priority=0,
|
||||
dont_filter=False, errback=None, cb_kwargs=None):
|
||||
dont_filter=False, errback=None, cb_kwargs=None, flags=None):
|
||||
# type: (...) -> Request
|
||||
"""
|
||||
Return a :class:`~.Request` instance to follow a link ``url``.
|
||||
|
|
@ -125,6 +138,7 @@ class Response(object_ref):
|
|||
elif url is None:
|
||||
raise ValueError("url can't be None")
|
||||
url = self.urljoin(url)
|
||||
|
||||
return Request(
|
||||
url=url,
|
||||
callback=callback,
|
||||
|
|
@ -138,11 +152,12 @@ class Response(object_ref):
|
|||
dont_filter=dont_filter,
|
||||
errback=errback,
|
||||
cb_kwargs=cb_kwargs,
|
||||
flags=flags,
|
||||
)
|
||||
|
||||
def follow_all(self, urls, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding='utf-8', priority=0,
|
||||
dont_filter=False, errback=None, cb_kwargs=None):
|
||||
dont_filter=False, errback=None, cb_kwargs=None, flags=None):
|
||||
# type: (...) -> Generator[Request, None, None]
|
||||
"""
|
||||
Return an iterable of :class:`~.Request` instances to follow all links
|
||||
|
|
@ -170,6 +185,7 @@ class Response(object_ref):
|
|||
dont_filter=dont_filter,
|
||||
errback=errback,
|
||||
cb_kwargs=cb_kwargs,
|
||||
flags=flags,
|
||||
)
|
||||
for url in urls
|
||||
)
|
||||
|
|
|
|||
|
|
@ -121,7 +121,7 @@ class TextResponse(Response):
|
|||
|
||||
def follow(self, url, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding=None, priority=0,
|
||||
dont_filter=False, errback=None, cb_kwargs=None):
|
||||
dont_filter=False, errback=None, cb_kwargs=None, flags=None):
|
||||
# type: (...) -> Request
|
||||
"""
|
||||
Return a :class:`~.Request` instance to follow a link ``url``.
|
||||
|
|
@ -157,11 +157,12 @@ class TextResponse(Response):
|
|||
dont_filter=dont_filter,
|
||||
errback=errback,
|
||||
cb_kwargs=cb_kwargs,
|
||||
flags=flags,
|
||||
)
|
||||
|
||||
def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
|
||||
cookies=None, meta=None, encoding=None, priority=0,
|
||||
dont_filter=False, errback=None, cb_kwargs=None,
|
||||
dont_filter=False, errback=None, cb_kwargs=None, flags=None,
|
||||
css=None, xpath=None):
|
||||
# type: (...) -> Generator[Request, None, None]
|
||||
"""
|
||||
|
|
@ -214,6 +215,7 @@ class TextResponse(Response):
|
|||
dont_filter=dont_filter,
|
||||
errback=errback,
|
||||
cb_kwargs=cb_kwargs,
|
||||
flags=flags,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,11 +5,11 @@ from urllib.parse import urljoin
|
|||
|
||||
import lxml.etree as etree
|
||||
from w3lib.html import strip_html5_whitespace
|
||||
from w3lib.url import canonicalize_url
|
||||
from w3lib.url import canonicalize_url, safe_url_string
|
||||
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
|
||||
from scrapy.utils.python import unique as unique_list, to_unicode
|
||||
from scrapy.utils.python import unique as unique_list
|
||||
from scrapy.utils.response import get_base_url
|
||||
from scrapy.linkextractors import FilteringLinkExtractor
|
||||
|
||||
|
|
@ -22,7 +22,7 @@ _collect_string_content = etree.XPath("string()")
|
|||
|
||||
def _nons(tag):
|
||||
if isinstance(tag, str):
|
||||
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
|
||||
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE:
|
||||
return tag.split('}')[-1]
|
||||
return tag
|
||||
|
||||
|
|
@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object):
|
|||
url = self.process_attr(attr_val)
|
||||
if url is None:
|
||||
continue
|
||||
url = to_unicode(url, encoding=response_encoding)
|
||||
url = safe_url_string(url, encoding=response_encoding)
|
||||
# to fix relative links after process_value
|
||||
url = urljoin(response_url, url)
|
||||
link = Link(url, _collect_string_content(el) or u'',
|
||||
|
|
|
|||
|
|
@ -5,10 +5,13 @@ from twisted.python.failure import Failure
|
|||
|
||||
from scrapy.utils.request import referer_str
|
||||
|
||||
SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s"
|
||||
DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
|
||||
CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
|
||||
ERRORMSG = u"'Error processing %(item)s'"
|
||||
SCRAPEDMSG = "Scraped from %(src)s" + os.linesep + "%(item)s"
|
||||
DROPPEDMSG = "Dropped: %(exception)s" + os.linesep + "%(item)s"
|
||||
CRAWLEDMSG = "Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s"
|
||||
ITEMERRORMSG = "Error processing %(item)s"
|
||||
SPIDERERRORMSG = "Spider error processing %(request)s (referer: %(referer)s)"
|
||||
DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
|
||||
DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
|
||||
|
||||
|
||||
class LogFormatter(object):
|
||||
|
|
@ -93,16 +96,41 @@ class LogFormatter(object):
|
|||
}
|
||||
}
|
||||
|
||||
def error(self, item, exception, response, spider):
|
||||
def item_error(self, item, exception, response, spider):
|
||||
"""Logs a message when an item causes an error while it is passing through the item pipeline."""
|
||||
return {
|
||||
'level': logging.ERROR,
|
||||
'msg': ERRORMSG,
|
||||
'msg': ITEMERRORMSG,
|
||||
'args': {
|
||||
'item': item,
|
||||
}
|
||||
}
|
||||
|
||||
def spider_error(self, failure, request, response, spider):
|
||||
"""Logs an error message from a spider."""
|
||||
return {
|
||||
'level': logging.ERROR,
|
||||
'msg': SPIDERERRORMSG,
|
||||
'args': {
|
||||
'request': request,
|
||||
'referer': referer_str(request),
|
||||
}
|
||||
}
|
||||
|
||||
def download_error(self, failure, request, spider, errmsg=None):
|
||||
"""Logs a download error message from a spider (typically coming from the engine)."""
|
||||
args = {'request': request}
|
||||
if errmsg:
|
||||
msg = DOWNLOADERRORMSG_LONG
|
||||
args['errmsg'] = errmsg
|
||||
else:
|
||||
msg = DOWNLOADERRORMSG_SHORT
|
||||
return {
|
||||
'level': logging.ERROR,
|
||||
'msg': msg,
|
||||
'args': args,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
return cls()
|
||||
|
|
|
|||
|
|
@ -1,11 +1,7 @@
|
|||
import hashlib
|
||||
import logging
|
||||
from collections import namedtuple
|
||||
|
||||
from queuelib import PriorityQueue
|
||||
|
||||
from scrapy.utils.reqser import request_to_dict, request_from_dict
|
||||
|
||||
from scrapy.utils.misc import create_instance
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -29,88 +25,89 @@ def _path_safe(text):
|
|||
return '-'.join([pathable_slot, unique_slot])
|
||||
|
||||
|
||||
class _Priority(namedtuple("_Priority", ["priority", "slot"])):
|
||||
""" Slot-specific priority. It is a hack - ``(priority, slot)`` tuple
|
||||
which can be used instead of int priorities in queues:
|
||||
class ScrapyPriorityQueue:
|
||||
"""A priority queue implemented using multiple internal queues (typically,
|
||||
FIFO queues). It uses one internal queue for each priority value. The internal
|
||||
queue must implement the following methods:
|
||||
|
||||
* push(obj)
|
||||
* pop()
|
||||
* close()
|
||||
* __len__()
|
||||
|
||||
``__init__`` method of ScrapyPriorityQueue receives a downstream_queue_cls
|
||||
argument, which is a class used to instantiate a new (internal) queue when
|
||||
a new priority is allocated.
|
||||
|
||||
Only integer priorities should be used. Lower numbers are higher
|
||||
priorities.
|
||||
|
||||
startprios is a sequence of priorities to start with. If the queue was
|
||||
previously closed leaving some priority buckets non-empty, those priorities
|
||||
should be passed in startprios.
|
||||
|
||||
* they are ordered in the same way - order is still by priority value,
|
||||
min(prios) works;
|
||||
* str(p) representation is guaranteed to be different when slots
|
||||
are different - this is important because str(p) is used to create
|
||||
queue files on disk;
|
||||
* they have readable str(p) representation which is safe
|
||||
to use as a file name.
|
||||
"""
|
||||
__slots__ = ()
|
||||
|
||||
def __str__(self):
|
||||
return '%s_%s' % (self.priority, _path_safe(str(self.slot)))
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
|
||||
return cls(crawler, downstream_queue_cls, key, startprios)
|
||||
|
||||
def __init__(self, crawler, downstream_queue_cls, key, startprios=()):
|
||||
self.crawler = crawler
|
||||
self.downstream_queue_cls = downstream_queue_cls
|
||||
self.key = key
|
||||
self.queues = {}
|
||||
self.curprio = None
|
||||
self.init_prios(startprios)
|
||||
|
||||
class _SlotPriorityQueues(object):
|
||||
""" Container for multiple priority queues. """
|
||||
def __init__(self, pqfactory, slot_startprios=None):
|
||||
"""
|
||||
``pqfactory`` is a factory for creating new PriorityQueues.
|
||||
It must be a function which accepts a single optional ``startprios``
|
||||
argument, with a list of priorities to create queues for.
|
||||
def init_prios(self, startprios):
|
||||
if not startprios:
|
||||
return
|
||||
|
||||
``slot_startprios`` is a ``{slot: startprios}`` dict.
|
||||
"""
|
||||
self.pqfactory = pqfactory
|
||||
self.pqueues = {} # slot -> priority queue
|
||||
for slot, startprios in (slot_startprios or {}).items():
|
||||
self.pqueues[slot] = self.pqfactory(startprios)
|
||||
for priority in startprios:
|
||||
self.queues[priority] = self.qfactory(priority)
|
||||
|
||||
def pop_slot(self, slot):
|
||||
""" Pop an object from a priority queue for this slot """
|
||||
queue = self.pqueues[slot]
|
||||
request = queue.pop()
|
||||
if len(queue) == 0:
|
||||
del self.pqueues[slot]
|
||||
return request
|
||||
self.curprio = min(startprios)
|
||||
|
||||
def push_slot(self, slot, obj, priority):
|
||||
""" Push an object to a priority queue for this slot """
|
||||
if slot not in self.pqueues:
|
||||
self.pqueues[slot] = self.pqfactory()
|
||||
queue = self.pqueues[slot]
|
||||
queue.push(obj, priority)
|
||||
def qfactory(self, key):
|
||||
return create_instance(self.downstream_queue_cls,
|
||||
None,
|
||||
self.crawler,
|
||||
self.key + '/' + str(key))
|
||||
|
||||
def priority(self, request):
|
||||
return -request.priority
|
||||
|
||||
def push(self, request):
|
||||
priority = self.priority(request)
|
||||
if priority not in self.queues:
|
||||
self.queues[priority] = self.qfactory(priority)
|
||||
q = self.queues[priority]
|
||||
q.push(request) # this may fail (eg. serialization error)
|
||||
if self.curprio is None or priority < self.curprio:
|
||||
self.curprio = priority
|
||||
|
||||
def pop(self):
|
||||
if self.curprio is None:
|
||||
return
|
||||
q = self.queues[self.curprio]
|
||||
m = q.pop()
|
||||
if not q:
|
||||
del self.queues[self.curprio]
|
||||
q.close()
|
||||
prios = [p for p, q in self.queues.items() if q]
|
||||
self.curprio = min(prios) if prios else None
|
||||
return m
|
||||
|
||||
def close(self):
|
||||
active = {slot: queue.close()
|
||||
for slot, queue in self.pqueues.items()}
|
||||
self.pqueues.clear()
|
||||
active = []
|
||||
for p, q in self.queues.items():
|
||||
active.append(p)
|
||||
q.close()
|
||||
return active
|
||||
|
||||
def __len__(self):
|
||||
return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
|
||||
|
||||
|
||||
class ScrapyPriorityQueue(PriorityQueue):
|
||||
"""
|
||||
PriorityQueue which works with scrapy.Request instances and
|
||||
can optionally convert them to/from dicts before/after putting to a queue.
|
||||
"""
|
||||
def __init__(self, crawler, qfactory, startprios=(), serialize=False):
|
||||
super(ScrapyPriorityQueue, self).__init__(qfactory, startprios)
|
||||
self.serialize = serialize
|
||||
self.spider = crawler.spider
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False):
|
||||
return cls(crawler, qfactory, startprios, serialize)
|
||||
|
||||
def push(self, request, priority=0):
|
||||
if self.serialize:
|
||||
request = request_to_dict(request, self.spider)
|
||||
super(ScrapyPriorityQueue, self).push(request, priority)
|
||||
|
||||
def pop(self):
|
||||
request = super(ScrapyPriorityQueue, self).pop()
|
||||
if request and self.serialize:
|
||||
request = request_from_dict(request, self.spider)
|
||||
return request
|
||||
return sum(len(x) for x in self.queues.values()) if self.queues else 0
|
||||
|
||||
|
||||
class DownloaderInterface(object):
|
||||
|
|
@ -133,16 +130,16 @@ class DownloaderInterface(object):
|
|||
|
||||
|
||||
class DownloaderAwarePriorityQueue(object):
|
||||
""" PriorityQueue which takes Downlaoder activity in account:
|
||||
""" PriorityQueue which takes Downloader activity in account:
|
||||
domains (slots) with the least amount of active downloads are dequeued
|
||||
first.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False):
|
||||
return cls(crawler, qfactory, slot_startprios, serialize)
|
||||
def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
|
||||
return cls(crawler, downstream_queue_cls, key, startprios)
|
||||
|
||||
def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False):
|
||||
def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
|
||||
if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0:
|
||||
raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP'
|
||||
% (self.__class__,))
|
||||
|
|
@ -156,35 +153,49 @@ class DownloaderAwarePriorityQueue(object):
|
|||
"queue class can be resumed." %
|
||||
slot_startprios.__class__)
|
||||
|
||||
slot_startprios = {
|
||||
slot: [_Priority(p, slot) for p in startprios]
|
||||
for slot, startprios in (slot_startprios or {}).items()}
|
||||
|
||||
def pqfactory(startprios=()):
|
||||
return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize)
|
||||
self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios)
|
||||
self.serialize = serialize
|
||||
self._downloader_interface = DownloaderInterface(crawler)
|
||||
self.downstream_queue_cls = downstream_queue_cls
|
||||
self.key = key
|
||||
self.crawler = crawler
|
||||
|
||||
self.pqueues = {} # slot -> priority queue
|
||||
for slot, startprios in (slot_startprios or {}).items():
|
||||
self.pqueues[slot] = self.pqfactory(slot, startprios)
|
||||
|
||||
def pqfactory(self, slot, startprios=()):
|
||||
return ScrapyPriorityQueue(self.crawler,
|
||||
self.downstream_queue_cls,
|
||||
self.key + '/' + _path_safe(slot),
|
||||
startprios)
|
||||
|
||||
def pop(self):
|
||||
stats = self._downloader_interface.stats(self._slot_pqueues.pqueues)
|
||||
stats = self._downloader_interface.stats(self.pqueues)
|
||||
|
||||
if not stats:
|
||||
return
|
||||
|
||||
slot = min(stats)[1]
|
||||
request = self._slot_pqueues.pop_slot(slot)
|
||||
queue = self.pqueues[slot]
|
||||
request = queue.pop()
|
||||
if len(queue) == 0:
|
||||
del self.pqueues[slot]
|
||||
return request
|
||||
|
||||
def push(self, request, priority):
|
||||
def push(self, request):
|
||||
slot = self._downloader_interface.get_slot_key(request)
|
||||
priority_slot = _Priority(priority=priority, slot=slot)
|
||||
self._slot_pqueues.push_slot(slot, request, priority_slot)
|
||||
if slot not in self.pqueues:
|
||||
self.pqueues[slot] = self.pqfactory(slot)
|
||||
queue = self.pqueues[slot]
|
||||
queue.push(request)
|
||||
|
||||
def close(self):
|
||||
active = self._slot_pqueues.close()
|
||||
return {slot: [p.priority for p in startprios]
|
||||
for slot, startprios in active.items()}
|
||||
active = {slot: queue.close()
|
||||
for slot, queue in self.pqueues.items()}
|
||||
self.pqueues.clear()
|
||||
return active
|
||||
|
||||
def __len__(self):
|
||||
return len(self._slot_pqueues)
|
||||
return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
|
||||
|
||||
def __contains__(self, slot):
|
||||
return slot in self.pqueues
|
||||
|
|
|
|||
|
|
@ -116,4 +116,5 @@ class ResponseTypes(object):
|
|||
cls = self.from_body(body)
|
||||
return cls
|
||||
|
||||
|
||||
responsetypes = ResponseTypes()
|
||||
|
|
|
|||
|
|
@ -75,8 +75,8 @@ DOWNLOAD_HANDLERS_BASE = {
|
|||
|
||||
DOWNLOAD_TIMEOUT = 180 # 3mins
|
||||
|
||||
DOWNLOAD_MAXSIZE = 1024*1024*1024 # 1024m
|
||||
DOWNLOAD_WARNSIZE = 32*1024*1024 # 32m
|
||||
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
|
||||
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
|
||||
|
||||
DOWNLOAD_FAIL_ON_DATALOSS = True
|
||||
|
||||
|
|
|
|||
|
|
@ -5,14 +5,13 @@ See documentation in docs/topics/shell.rst
|
|||
"""
|
||||
import os
|
||||
import signal
|
||||
import warnings
|
||||
|
||||
from twisted.internet import threads, defer
|
||||
from twisted.python import threadable
|
||||
from w3lib.url import any_to_uri
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.http import Request, Response
|
||||
from scrapy.item import BaseItem
|
||||
from scrapy.settings import Settings
|
||||
|
|
@ -126,7 +125,6 @@ class Shell(object):
|
|||
self.vars['spider'] = spider
|
||||
self.vars['request'] = request
|
||||
self.vars['response'] = response
|
||||
self.vars['sel'] = _SelectorProxy(response)
|
||||
if self.inthread:
|
||||
self.vars['fetch'] = self.fetch
|
||||
self.vars['view'] = open_in_browser
|
||||
|
|
@ -173,7 +171,7 @@ def _request_deferred(request):
|
|||
|
||||
This returns a Deferred whose first pair of callbacks are the request
|
||||
callback and errback. The Deferred also triggers when the request
|
||||
callback/errback is executed (ie. when the request is downloaded)
|
||||
callback/errback is executed (i.e. when the request is downloaded)
|
||||
|
||||
WARNING: Do not call request.replace() until after the deferred is called.
|
||||
"""
|
||||
|
|
@ -192,15 +190,3 @@ def _request_deferred(request):
|
|||
|
||||
request.callback, request.errback = d.callback, d.errback
|
||||
return d
|
||||
|
||||
|
||||
class _SelectorProxy(object):
|
||||
|
||||
def __init__(self, response):
|
||||
self._proxiedresponse = response
|
||||
|
||||
def __getattr__(self, name):
|
||||
warnings.warn('"sel" shortcut is deprecated. Use "response.xpath()", '
|
||||
'"response.css()" or "response.selector" instead',
|
||||
category=ScrapyDeprecationWarning, stacklevel=2)
|
||||
return getattr(self._proxiedresponse.selector, name)
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ class SpiderLoader(object):
|
|||
module=mod, cls=cls, name=name)
|
||||
for (mod, cls) in locations)
|
||||
for name, locations in self._found.items()
|
||||
if len(locations)>1]
|
||||
if len(locations) > 1]
|
||||
if dupes:
|
||||
msg = ("There are several spiders with the same name:\n\n"
|
||||
"{}\n\n This can cause unexpected behavior.".format(
|
||||
|
|
|
|||
|
|
@ -3,10 +3,27 @@ Scheduler queues
|
|||
"""
|
||||
|
||||
import marshal
|
||||
import os
|
||||
import pickle
|
||||
|
||||
from queuelib import queue
|
||||
|
||||
from scrapy.utils.reqser import request_to_dict, request_from_dict
|
||||
|
||||
|
||||
def _with_mkdir(queue_class):
|
||||
|
||||
class DirectoriesCreated(queue_class):
|
||||
|
||||
def __init__(self, path, *args, **kwargs):
|
||||
dirname = os.path.dirname(path)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname, exist_ok=True)
|
||||
|
||||
super(DirectoriesCreated, self).__init__(path, *args, **kwargs)
|
||||
|
||||
return DirectoriesCreated
|
||||
|
||||
|
||||
def _serializable_queue(queue_class, serialize, deserialize):
|
||||
|
||||
|
|
@ -24,6 +41,44 @@ def _serializable_queue(queue_class, serialize, deserialize):
|
|||
return SerializableQueue
|
||||
|
||||
|
||||
def _scrapy_serialization_queue(queue_class):
|
||||
|
||||
class ScrapyRequestQueue(queue_class):
|
||||
|
||||
def __init__(self, crawler, key):
|
||||
self.spider = crawler.spider
|
||||
super(ScrapyRequestQueue, self).__init__(key)
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, key, *args, **kwargs):
|
||||
return cls(crawler, key)
|
||||
|
||||
def push(self, request):
|
||||
request = request_to_dict(request, self.spider)
|
||||
return super(ScrapyRequestQueue, self).push(request)
|
||||
|
||||
def pop(self):
|
||||
request = super(ScrapyRequestQueue, self).pop()
|
||||
|
||||
if not request:
|
||||
return None
|
||||
|
||||
request = request_from_dict(request, self.spider)
|
||||
return request
|
||||
|
||||
return ScrapyRequestQueue
|
||||
|
||||
|
||||
def _scrapy_non_serialization_queue(queue_class):
|
||||
|
||||
class ScrapyRequestQueue(queue_class):
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
return cls()
|
||||
|
||||
return ScrapyRequestQueue
|
||||
|
||||
|
||||
def _pickle_serialize(obj):
|
||||
try:
|
||||
return pickle.dumps(obj, protocol=2)
|
||||
|
|
@ -34,13 +89,38 @@ def _pickle_serialize(obj):
|
|||
raise ValueError(str(e))
|
||||
|
||||
|
||||
PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
|
||||
_pickle_serialize, pickle.loads)
|
||||
PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
|
||||
_pickle_serialize, pickle.loads)
|
||||
MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue,
|
||||
marshal.dumps, marshal.loads)
|
||||
MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue,
|
||||
marshal.dumps, marshal.loads)
|
||||
FifoMemoryQueue = queue.FifoMemoryQueue
|
||||
LifoMemoryQueue = queue.LifoMemoryQueue
|
||||
PickleFifoDiskQueueNonRequest = _serializable_queue(
|
||||
_with_mkdir(queue.FifoDiskQueue),
|
||||
_pickle_serialize,
|
||||
pickle.loads
|
||||
)
|
||||
PickleLifoDiskQueueNonRequest = _serializable_queue(
|
||||
_with_mkdir(queue.LifoDiskQueue),
|
||||
_pickle_serialize,
|
||||
pickle.loads
|
||||
)
|
||||
MarshalFifoDiskQueueNonRequest = _serializable_queue(
|
||||
_with_mkdir(queue.FifoDiskQueue),
|
||||
marshal.dumps,
|
||||
marshal.loads
|
||||
)
|
||||
MarshalLifoDiskQueueNonRequest = _serializable_queue(
|
||||
_with_mkdir(queue.LifoDiskQueue),
|
||||
marshal.dumps,
|
||||
marshal.loads
|
||||
)
|
||||
|
||||
PickleFifoDiskQueue = _scrapy_serialization_queue(
|
||||
PickleFifoDiskQueueNonRequest
|
||||
)
|
||||
PickleLifoDiskQueue = _scrapy_serialization_queue(
|
||||
PickleLifoDiskQueueNonRequest
|
||||
)
|
||||
MarshalFifoDiskQueue = _scrapy_serialization_queue(
|
||||
MarshalFifoDiskQueueNonRequest
|
||||
)
|
||||
MarshalLifoDiskQueue = _scrapy_serialization_queue(
|
||||
MarshalLifoDiskQueueNonRequest
|
||||
)
|
||||
FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
|
||||
LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ def _embed_standard_shell(namespace={}, banner=''):
|
|||
else:
|
||||
import rlcompleter # noqa: F401
|
||||
readline.parse_and_bind("tab:complete")
|
||||
|
||||
@wraps(_embed_standard_shell)
|
||||
def wrapper(namespace=namespace, banner=''):
|
||||
code.interact(banner=banner, local=namespace)
|
||||
|
|
|
|||
|
|
@ -175,12 +175,12 @@ class SiteNode(object):
|
|||
node.parent = self
|
||||
|
||||
def to_string(self, level=0):
|
||||
s = "%s%s\n" % (' '*level, self.url)
|
||||
s = "%s%s\n" % (' ' * level, self.url)
|
||||
if self.itemnames:
|
||||
for n in self.itemnames:
|
||||
s += "%sScraped: %s\n" % (' '*(level+1), n)
|
||||
s += "%sScraped: %s\n" % (' ' * (level + 1), n)
|
||||
for node in self.children:
|
||||
s += node.to_string(level+1)
|
||||
s += node.to_string(level + 1)
|
||||
return s
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -153,3 +153,20 @@ def deferred_f_from_coro_f(coro_f):
|
|||
def f(*coro_args, **coro_kwargs):
|
||||
return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
|
||||
return f
|
||||
|
||||
|
||||
def maybeDeferred_coro(f, *args, **kw):
|
||||
""" Copy of defer.maybeDeferred that also converts coroutines to Deferreds. """
|
||||
try:
|
||||
result = f(*args, **kw)
|
||||
except: # noqa: E722
|
||||
return defer.fail(failure.Failure(captureVars=defer.Deferred.debug))
|
||||
|
||||
if isinstance(result, defer.Deferred):
|
||||
return result
|
||||
elif _isfuture(result) or inspect.isawaitable(result):
|
||||
return deferred_from_coro(result)
|
||||
elif isinstance(result, failure.Failure):
|
||||
return defer.fail(result)
|
||||
else:
|
||||
return defer.succeed(result)
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ def gunzip(data):
|
|||
raise
|
||||
return b''.join(output_list)
|
||||
|
||||
|
||||
_is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search
|
||||
_is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search
|
||||
|
||||
|
|
|
|||
|
|
@ -32,5 +32,5 @@ def decode_chunked_transfer(chunked_body):
|
|||
break
|
||||
size = int(h, 16)
|
||||
body += t[:size]
|
||||
t = t[size+2:]
|
||||
t = t[size + 2:]
|
||||
return body
|
||||
|
|
|
|||
|
|
@ -101,8 +101,10 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
|
|||
lines = StringIO(_body_or_str(obj, unicode=True))
|
||||
|
||||
kwargs = {}
|
||||
if delimiter: kwargs["delimiter"] = delimiter
|
||||
if quotechar: kwargs["quotechar"] = quotechar
|
||||
if delimiter:
|
||||
kwargs["delimiter"] = delimiter
|
||||
if quotechar:
|
||||
kwargs["quotechar"] = quotechar
|
||||
csv_r = csv.reader(lines, **kwargs)
|
||||
|
||||
if not headers:
|
||||
|
|
|
|||
|
|
@ -37,8 +37,8 @@ def arg_to_iter(arg):
|
|||
def load_object(path):
|
||||
"""Load an object given its absolute object path, and return it.
|
||||
|
||||
object can be a class, function, variable or an instance.
|
||||
path ie: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
|
||||
object can be the import path of a class, function, variable or an
|
||||
instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
|
||||
"""
|
||||
|
||||
try:
|
||||
|
|
@ -46,7 +46,7 @@ def load_object(path):
|
|||
except ValueError:
|
||||
raise ValueError("Error loading object '%s': not a full path" % path)
|
||||
|
||||
module, name = path[:dot], path[dot+1:]
|
||||
module, name = path[:dot], path[dot + 1:]
|
||||
mod = import_module(module)
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ def get_project_settings():
|
|||
if settings_module_path:
|
||||
settings.setmodule(settings_module_path, priority='project')
|
||||
|
||||
# XXX: remove this hack
|
||||
pickled_settings = os.environ.get("SCRAPY_PICKLED_SETTINGS_TO_OVERRIDE")
|
||||
if pickled_settings:
|
||||
warnings.warn("Use of environment variable "
|
||||
|
|
@ -76,10 +75,9 @@ def get_project_settings():
|
|||
"is deprecated.", ScrapyDeprecationWarning)
|
||||
settings.setdict(pickle.loads(pickled_settings), priority='project')
|
||||
|
||||
# XXX: deprecate and remove this functionality
|
||||
env_overrides = {k[7:]: v for k, v in os.environ.items() if
|
||||
k.startswith('SCRAPY_')}
|
||||
if env_overrides:
|
||||
warnings.warn("Use of 'SCRAPY_'-prefixed environment variables to override settings is deprecated.", ScrapyDeprecationWarning)
|
||||
settings.setdict(env_overrides, priority='project')
|
||||
|
||||
return settings
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
"""
|
||||
Helpers using Python 3.6+ syntax (ignore SyntaxError on import).
|
||||
"""
|
||||
|
||||
|
||||
async def collect_asyncgen(result):
|
||||
results = []
|
||||
async for x in result:
|
||||
results.append(x)
|
||||
return results
|
||||
|
|
@ -4,7 +4,6 @@ This module contains essential stuff that should've come with Python itself ;)
|
|||
import errno
|
||||
import gc
|
||||
import inspect
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import weakref
|
||||
|
|
@ -165,14 +164,6 @@ _BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\
|
|||
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
|
||||
|
||||
|
||||
@deprecated("scrapy.utils.python.binary_is_text")
|
||||
def isbinarytext(text):
|
||||
""" This function is deprecated.
|
||||
Please use scrapy.utils.python.binary_is_text, which was created to be more
|
||||
clear about the functions behavior: it is behaving inverted to this one. """
|
||||
return not binary_is_text(text)
|
||||
|
||||
|
||||
def binary_is_text(data):
|
||||
""" Returns ``True`` if the given ``data`` argument (a ``bytes`` object)
|
||||
does not contain unprintable control characters.
|
||||
|
|
@ -293,41 +284,6 @@ class WeakKeyCache(object):
|
|||
return self._weakdict[key]
|
||||
|
||||
|
||||
@deprecated
|
||||
def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
|
||||
"""Return a (new) dict with unicode keys (and values when "keys_only" is
|
||||
False) of the given dict converted to strings. ``dct_or_tuples`` can be a
|
||||
dict or a list of tuples, like any dict ``__init__`` method supports.
|
||||
"""
|
||||
d = {}
|
||||
for k, v in dict(dct_or_tuples).items():
|
||||
k = k.encode(encoding) if isinstance(k, str) else k
|
||||
if not keys_only:
|
||||
v = v.encode(encoding) if isinstance(v, str) else v
|
||||
d[k] = v
|
||||
return d
|
||||
|
||||
|
||||
@deprecated
|
||||
def is_writable(path):
|
||||
"""Return True if the given path can be written (if it exists) or created
|
||||
(if it doesn't exist)
|
||||
"""
|
||||
if os.path.exists(path):
|
||||
return os.access(path, os.W_OK)
|
||||
else:
|
||||
return os.access(os.path.dirname(path), os.W_OK)
|
||||
|
||||
|
||||
@deprecated
|
||||
def setattr_default(obj, name, value):
|
||||
"""Set attribute value, but only if it's not already set. Similar to
|
||||
setdefault() for dicts.
|
||||
"""
|
||||
if not hasattr(obj, name):
|
||||
setattr(obj, name, value)
|
||||
|
||||
|
||||
def retry_on_eintr(function, *args, **kw):
|
||||
"""Run a function and retry it while getting EINTR errors"""
|
||||
while True:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ def listen_tcp(portrange, host, factory):
|
|||
return reactor.listenTCP(portrange, factory, interface=host)
|
||||
if len(portrange) == 1:
|
||||
return reactor.listenTCP(portrange[0], factory, interface=host)
|
||||
for x in range(portrange[0], portrange[1]+1):
|
||||
for x in range(portrange[0], portrange[1] + 1):
|
||||
try:
|
||||
return reactor.listenTCP(x, factory, interface=host)
|
||||
except error.CannotListenError:
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ def request_fingerprint(request, include_headers=None, keep_fragments=False):
|
|||
http://www.example.com/query?cat=222&id=111
|
||||
|
||||
Even though those are two different URLs both point to the same resource
|
||||
and are equivalent (ie. they should return the same response).
|
||||
and are equivalent (i.e. they should return the same response).
|
||||
|
||||
Another example are cookies used to store session ids. Suppose the
|
||||
following page is only accessible to authenticated users:
|
||||
|
|
|
|||
|
|
@ -2,12 +2,14 @@
|
|||
|
||||
import logging
|
||||
|
||||
from twisted.internet.defer import maybeDeferred, DeferredList, Deferred
|
||||
from twisted.internet.defer import DeferredList, Deferred
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from pydispatch.dispatcher import Any, Anonymous, liveReceivers, \
|
||||
getAllReceivers, disconnect
|
||||
from pydispatch.robustapply import robustApply
|
||||
|
||||
from scrapy.utils.defer import maybeDeferred_coro
|
||||
from scrapy.utils.log import failure_to_exc_info
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -61,7 +63,7 @@ def send_catch_log_deferred(signal=Any, sender=Anonymous, *arguments, **named):
|
|||
spider = named.get('spider', None)
|
||||
dfds = []
|
||||
for receiver in liveReceivers(getAllReceivers(sender, signal)):
|
||||
d = maybeDeferred(robustApply, receiver, signal=signal, sender=sender,
|
||||
d = maybeDeferred_coro(robustApply, receiver, signal=signal, sender=sender,
|
||||
*arguments, **named)
|
||||
d.addErrback(logerror, receiver)
|
||||
d.addBoth(lambda result: (receiver, result))
|
||||
|
|
|
|||
|
|
@ -4,18 +4,26 @@ import inspect
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.defer import deferred_from_coro
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
try:
|
||||
from scrapy.utils.py36 import collect_asyncgen
|
||||
except SyntaxError:
|
||||
collect_asyncgen = None
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def iterate_spider_output(result):
|
||||
if collect_asyncgen and hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(result):
|
||||
d = deferred_from_coro(collect_asyncgen(result))
|
||||
d.addCallback(iterate_spider_output)
|
||||
return d
|
||||
return arg_to_iter(deferred_from_coro(result))
|
||||
|
||||
|
||||
def iter_spider_classes(module):
|
||||
"""Return an iterator over all spider classes defined in the given module
|
||||
that can be instantiated (ie. which have name)
|
||||
that can be instantiated (i.e. which have name)
|
||||
"""
|
||||
# this needs to be imported here until get rid of the spider manager
|
||||
# singleton in scrapy.spider.spiders
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ Prerequisites
|
|||
|
||||
This API proposal relies on the following API:
|
||||
|
||||
1. instantiating a item with an item instance as its first argument (ie.
|
||||
1. instantiating a item with an item instance as its first argument (i.e.
|
||||
``item2 = MyItem(item1)``) must return a **copy** of the first item
|
||||
instance)
|
||||
2. items can be instantiated using this syntax: ``item = Item(attr1=value1,
|
||||
|
|
@ -78,7 +78,7 @@ Defining an item containing ItemField's
|
|||
variants2 = ListField(ItemField(Variant), default=[])
|
||||
|
||||
It's important to note here that the (perhaps most intuitive) way of defining a
|
||||
Product-Variant relationship (ie. defining a recursive !ItemField) doesn't
|
||||
Product-Variant relationship (i.e. defining a recursive !ItemField) doesn't
|
||||
work. For example, this fails to compile:
|
||||
|
||||
::
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ Global changes to all middlewares
|
|||
|
||||
To be discussed:
|
||||
|
||||
1. should we support returning deferreds (ie. ``maybeDeferred``) in middleware
|
||||
1. should we support returning deferreds (i.e. ``maybeDeferred``) in middleware
|
||||
methods?
|
||||
2. should we pass Twisted Failures instead of exceptions to error methods?
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ Goals:
|
|||
|
||||
* simple to manage: adding or removing extensions should be just a matter of
|
||||
adding or removing lines in a ``scrapy.cfg`` file
|
||||
* backward compatibility with enabling extension the "old way" (ie. modifying
|
||||
* backward compatibility with enabling extension the "old way" (i.e. modifying
|
||||
settings directly)
|
||||
|
||||
Non-goals:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Some pipelines used for testing
|
|||
class ZeroDivisionErrorPipeline(object):
|
||||
|
||||
def open_spider(self, spider):
|
||||
a = 1/0
|
||||
a = 1 / 0
|
||||
|
||||
def process_item(self, item, spider):
|
||||
return item
|
||||
|
|
@ -15,4 +15,4 @@ class ZeroDivisionErrorPipeline(object):
|
|||
class ProcessWithZeroDivisionErrorPipiline(object):
|
||||
|
||||
def process_item(self, item, spider):
|
||||
1/0
|
||||
1 / 0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
import asyncio
|
||||
|
||||
from scrapy import Request
|
||||
from tests.spiders import SimpleSpider
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen'
|
||||
|
||||
async def parse(self, response):
|
||||
await asyncio.sleep(0.2)
|
||||
yield {'foo': 42}
|
||||
self.logger.info("Got response %d" % response.status)
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenLoopSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen_loop'
|
||||
|
||||
async def parse(self, response):
|
||||
for i in range(10):
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'foo': i}
|
||||
self.logger.info("Got response %d" % response.status)
|
||||
|
||||
|
||||
class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
|
||||
|
||||
name = 'asyncdef_asyncio_gen_complex'
|
||||
initial_reqs = 4
|
||||
following_reqs = 3
|
||||
depth = 2
|
||||
|
||||
def _get_req(self, index, cb=None):
|
||||
return Request(self.mockserver.url("/status?n=200&request=%d" % index),
|
||||
meta={'index': index},
|
||||
dont_filter=True,
|
||||
callback=cb)
|
||||
|
||||
def start_requests(self):
|
||||
for i in range(1, self.initial_reqs + 1):
|
||||
yield self._get_req(i)
|
||||
|
||||
async def parse(self, response):
|
||||
index = response.meta['index']
|
||||
yield {'index': index}
|
||||
if index < 10 ** self.depth:
|
||||
for new_index in range(10 * index, 10 * index + self.following_reqs):
|
||||
yield self._get_req(new_index)
|
||||
yield self._get_req(index, cb=self.parse2)
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'index': index + 5}
|
||||
|
||||
async def parse2(self, response):
|
||||
await asyncio.sleep(0.1)
|
||||
yield {'index2': response.meta['index']}
|
||||
|
|
@ -147,7 +147,6 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
|||
self.url('/html')])
|
||||
self.assertIn("DEBUG: It Works!", _textmode(stderr))
|
||||
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_pipelines(self):
|
||||
_, _, stderr = yield self.execute(['--spider', self.spider_name,
|
||||
|
|
@ -183,7 +182,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
|||
def test_crawlspider_matching_rule_callback_set(self):
|
||||
"""If a rule matches the URL, use it's defined callback."""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/html')]
|
||||
['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/html')]
|
||||
)
|
||||
self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
|
||||
|
||||
|
|
@ -191,7 +190,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
|||
def test_crawlspider_matching_rule_default_callback(self):
|
||||
"""If a rule match but it has no callback set, use the 'parse' callback."""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'goodcrawl'+self.spider_name, '-r', self.url('/text')]
|
||||
['--spider', 'goodcrawl' + self.spider_name, '-r', self.url('/text')]
|
||||
)
|
||||
self.assertIn("""[{}, {'nomatch': 'default'}]""", _textmode(out))
|
||||
|
||||
|
|
@ -207,7 +206,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
|||
@defer.inlineCallbacks
|
||||
def test_crawlspider_missing_callback(self):
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/html')]
|
||||
['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/html')]
|
||||
)
|
||||
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
|
||||
|
||||
|
|
@ -215,7 +214,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
|
|||
def test_crawlspider_no_matching_rule(self):
|
||||
"""The requested URL has no matching rule, so no items should be scraped"""
|
||||
status, out, stderr = yield self.execute(
|
||||
['--spider', 'badcrawl'+self.spider_name, '-r', self.url('/enc-gb18030')]
|
||||
['--spider', 'badcrawl' + self.spider_name, '-r', self.url('/enc-gb18030')]
|
||||
)
|
||||
self.assertRegex(_textmode(out), r"""# Scraped Items -+\n\[\]""")
|
||||
self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
import logging
|
||||
import sys
|
||||
from ipaddress import IPv4Address
|
||||
from socket import gethostbyname
|
||||
from urllib.parse import urlparse
|
||||
|
|
@ -7,6 +8,7 @@ from urllib.parse import urlparse
|
|||
from pytest import mark
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.internet.ssl import Certificate
|
||||
from twisted.trial.unittest import TestCase
|
||||
|
||||
from scrapy import signals
|
||||
|
|
@ -348,6 +350,59 @@ with multiples lines
|
|||
self.assertIn({'id': 1}, items)
|
||||
self.assertIn({'id': 2}, items)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncgen_parse(self):
|
||||
from tests.py36._test_crawl import AsyncDefAsyncioGenSpider
|
||||
crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
|
||||
self.assertIn("Got response 200", str(log))
|
||||
itemcount = crawler.stats.get_value('item_scraped_count')
|
||||
self.assertEqual(itemcount, 1)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_loop(self):
|
||||
items = []
|
||||
|
||||
def _on_item_scraped(item):
|
||||
items.append(item)
|
||||
|
||||
from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider
|
||||
crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider)
|
||||
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
|
||||
with LogCapture() as log:
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
|
||||
self.assertIn("Got response 200", str(log))
|
||||
itemcount = crawler.stats.get_value('item_scraped_count')
|
||||
self.assertEqual(itemcount, 10)
|
||||
for i in range(10):
|
||||
self.assertIn({'foo': i}, items)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncgen_parse_complex(self):
|
||||
items = []
|
||||
|
||||
def _on_item_scraped(item):
|
||||
items.append(item)
|
||||
|
||||
from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider
|
||||
crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider)
|
||||
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
itemcount = crawler.stats.get_value('item_scraped_count')
|
||||
self.assertEqual(itemcount, 156)
|
||||
# some random items
|
||||
for i in [1, 4, 21, 22, 207, 311]:
|
||||
self.assertIn({'index': i}, items)
|
||||
for i in [10, 30, 122]:
|
||||
self.assertIn({'index2': i}, items)
|
||||
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_async_def_asyncio_parse_reqs_list(self):
|
||||
|
|
@ -357,6 +412,34 @@ with multiples lines
|
|||
for req_id in range(3):
|
||||
self.assertIn("Got response 200, req_id %d" % req_id, str(log))
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_ssl_certificate_none(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url("/echo?body=test", is_secure=False)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_response_ssl_certificate(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url("/echo?body=test", is_secure=True)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
cert = crawler.spider.meta['responses'][0].certificate
|
||||
self.assertIsInstance(cert, Certificate)
|
||||
self.assertEqual(cert.getSubject().commonName, b"localhost")
|
||||
self.assertEqual(cert.getIssuer().commonName, b"localhost")
|
||||
|
||||
@mark.xfail(reason="Responses with no body return early and contain no certificate")
|
||||
@defer.inlineCallbacks
|
||||
def test_response_ssl_certificate_empty_response(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url("/status?n=200", is_secure=True)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
cert = crawler.spider.meta['responses'][0].certificate
|
||||
self.assertIsInstance(cert, Certificate)
|
||||
self.assertEqual(cert.getSubject().commonName, b"localhost")
|
||||
self.assertEqual(cert.getIssuer().commonName, b"localhost")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_dns_server_ip_address(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
|
|
|
|||
|
|
@ -13,5 +13,6 @@ class ScrapyUtilsTest(unittest.TestCase):
|
|||
installed_version = [int(x) for x in module.__version__.split('.')[:2]]
|
||||
assert installed_version >= [0, 6], "OpenSSL >= 0.6 required"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -348,7 +348,7 @@ class HttpTestCase(unittest.TestCase):
|
|||
return self.download_request(request, Spider('foo')).addCallback(_test)
|
||||
|
||||
def test_payload(self):
|
||||
body = b'1'*100 # PayloadResource requires body length to be 100
|
||||
body = b'1' * 100 # PayloadResource requires body length to be 100
|
||||
request = Request(self.getURL('payload'), method='POST', body=body)
|
||||
d = self.download_request(request, Spider('foo'))
|
||||
d.addCallback(lambda r: r.body)
|
||||
|
|
@ -812,7 +812,7 @@ class S3TestCase(unittest.TestCase):
|
|||
|
||||
def test_request_signing1(self):
|
||||
# gets an object from the johnsmith bucket.
|
||||
date ='Tue, 27 Mar 2007 19:36:42 +0000'
|
||||
date = 'Tue, 27 Mar 2007 19:36:42 +0000'
|
||||
req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
|
||||
with self._mocked_date(date):
|
||||
httpreq = self.download_request(req, self.spider)
|
||||
|
|
|
|||
|
|
@ -145,7 +145,6 @@ class CookiesMiddlewareTest(TestCase):
|
|||
{'name': 'C3', 'value': 'value3', 'path': '/foo', 'domain': 'scrapytest.org'},
|
||||
{'name': 'C4', 'value': 'value4', 'path': '/foo', 'domain': 'scrapy.org'}]
|
||||
|
||||
|
||||
req = Request('http://scrapytest.org/', cookies=cookies)
|
||||
self.mw.process_request(req, self.spider)
|
||||
|
||||
|
|
|
|||
|
|
@ -501,5 +501,6 @@ class RFC2616PolicyTest(DefaultStorageTest):
|
|||
self.assertEqualResponse(res1, res2)
|
||||
assert 'cached' in res2.flags
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -245,7 +245,7 @@ class HttpCompressionTest(TestCase):
|
|||
response.headers['Content-Type'] = 'application/gzip'
|
||||
request = response.request
|
||||
request.method = 'HEAD'
|
||||
response = response.replace(body = None)
|
||||
response = response.replace(body=None)
|
||||
newresponse = self.mw.process_response(request, response, self.spider)
|
||||
self.assertIs(newresponse, response)
|
||||
self.assertEqual(response.body, b'')
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
assert isinstance(r, Response)
|
||||
assert r is rsp
|
||||
|
||||
|
||||
def test_redirect_302(self):
|
||||
url = 'http://www.example.com/302'
|
||||
url2 = 'http://www.example.com/redirected2'
|
||||
|
|
@ -122,7 +121,6 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
del rsp.headers['Location']
|
||||
assert self.mw.process_response(req, rsp, self.spider) is rsp
|
||||
|
||||
|
||||
def test_max_redirect_times(self):
|
||||
self.mw.max_redirect_times = 1
|
||||
req = Request('http://scrapytest.org/302')
|
||||
|
|
@ -178,6 +176,7 @@ class RedirectMiddlewareTest(unittest.TestCase):
|
|||
def test_request_meta_handling(self):
|
||||
url = 'http://www.example.com/301'
|
||||
url2 = 'http://www.example.com/redirected'
|
||||
|
||||
def _test_passthrough(req):
|
||||
rsp = Response(url, headers={'Location': url2}, status=301, request=req)
|
||||
r = self.mw.process_response(req, rsp, self.spider)
|
||||
|
|
@ -316,5 +315,6 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
|
|||
response = mw.process_response(req, rsp, self.spider)
|
||||
assert isinstance(response, Response)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
def test_df_from_crawler_scheduler(self):
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertTrue(scheduler.df.debug)
|
||||
|
|
@ -51,14 +51,14 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
|
||||
def test_df_from_settings_scheduler(self):
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertTrue(scheduler.df.debug)
|
||||
self.assertEqual(scheduler.df.method, 'from_settings')
|
||||
|
||||
def test_df_direct_scheduler(self):
|
||||
settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
|
||||
settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'}
|
||||
crawler = get_crawler(settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
self.assertEqual(scheduler.df.method, 'n/a')
|
||||
|
|
@ -162,7 +162,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_log(self):
|
||||
with LogCapture() as l:
|
||||
settings = {'DUPEFILTER_DEBUG': False,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
|
@ -187,7 +187,7 @@ class RFPDupeFilterTest(unittest.TestCase):
|
|||
def test_log_debug(self):
|
||||
with LogCapture() as l:
|
||||
settings = {'DUPEFILTER_DEBUG': True,
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
|
||||
crawler = get_crawler(SimpleSpider, settings_dict=settings)
|
||||
scheduler = Scheduler.from_crawler(crawler)
|
||||
spider = SimpleSpider.from_crawler(crawler)
|
||||
|
|
|
|||
|
|
@ -312,6 +312,7 @@ class XmlItemExporterTest(BaseItemExporterTest):
|
|||
for child in children]
|
||||
else:
|
||||
return [(elem.tag, [(elem.text, ())])]
|
||||
|
||||
def xmlsplit(xmlcontent):
|
||||
doc = lxml.etree.fromstring(xmlcontent)
|
||||
return xmltuple(doc)
|
||||
|
|
|
|||
|
|
@ -72,6 +72,22 @@ class BaseResponseTest(unittest.TestCase):
|
|||
r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
|
||||
assert r1.meta is req.meta
|
||||
|
||||
def test_copy_cb_kwargs(self):
|
||||
req = Request("http://www.example.com")
|
||||
req.cb_kwargs['foo'] = 'bar'
|
||||
r1 = self.response_class("http://www.example.com", body=b"Some body", request=req)
|
||||
assert r1.cb_kwargs is req.cb_kwargs
|
||||
|
||||
def test_unavailable_meta(self):
|
||||
r1 = self.response_class("http://www.example.com", body=b"Some body")
|
||||
with self.assertRaisesRegex(AttributeError, r'Response\.meta not available'):
|
||||
r1.meta
|
||||
|
||||
def test_unavailable_cb_kwargs(self):
|
||||
r1 = self.response_class("http://www.example.com", body=b"Some body")
|
||||
with self.assertRaisesRegex(AttributeError, r'Response\.cb_kwargs not available'):
|
||||
r1.cb_kwargs
|
||||
|
||||
def test_copy_inherited_classes(self):
|
||||
"""Test Response children copies preserve their class"""
|
||||
|
||||
|
|
@ -167,6 +183,11 @@ class BaseResponseTest(unittest.TestCase):
|
|||
self._assert_followed_url(Link('http://example.com/foo '),
|
||||
'http://example.com/foo%20')
|
||||
|
||||
def test_follow_flags(self):
|
||||
res = self.response_class('http://example.com/')
|
||||
fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
|
||||
self.assertEqual(fol.flags, ['cached', 'allowed'])
|
||||
|
||||
# Response.follow_all
|
||||
|
||||
def test_follow_all_absolute(self):
|
||||
|
|
@ -232,6 +253,17 @@ class BaseResponseTest(unittest.TestCase):
|
|||
expected = [u.replace(' ', '%20') for u in absolute]
|
||||
self._assert_followed_all_urls(links, expected)
|
||||
|
||||
def test_follow_all_flags(self):
|
||||
re = self.response_class('http://www.example.com/')
|
||||
urls = [
|
||||
'http://www.example.com/',
|
||||
'http://www.example.com/2',
|
||||
'http://www.example.com/foo',
|
||||
]
|
||||
fol = re.follow_all(urls, flags=['cached', 'allowed'])
|
||||
for req in fol:
|
||||
self.assertEqual(req.flags, ['cached', 'allowed'])
|
||||
|
||||
def _assert_followed_url(self, follow_obj, target_url, response=None):
|
||||
if response is None:
|
||||
response = self._links_response()
|
||||
|
|
@ -562,6 +594,22 @@ class TextResponseTest(BaseResponseTest):
|
|||
)
|
||||
self.assertEqual(req.encoding, 'cp1251')
|
||||
|
||||
def test_follow_flags(self):
|
||||
res = self.response_class('http://example.com/')
|
||||
fol = res.follow('http://example.com/', flags=['cached', 'allowed'])
|
||||
self.assertEqual(fol.flags, ['cached', 'allowed'])
|
||||
|
||||
def test_follow_all_flags(self):
|
||||
re = self.response_class('http://www.example.com/')
|
||||
urls = [
|
||||
'http://www.example.com/',
|
||||
'http://www.example.com/2',
|
||||
'http://www.example.com/foo',
|
||||
]
|
||||
fol = re.follow_all(urls, flags=['cached', 'allowed'])
|
||||
for req in fol:
|
||||
self.assertEqual(req.flags, ['cached', 'allowed'])
|
||||
|
||||
def test_follow_all_css(self):
|
||||
expected = [
|
||||
'http://example.com/sample3.html',
|
||||
|
|
|
|||
|
|
@ -149,13 +149,15 @@ class ItemTest(unittest.TestCase):
|
|||
fields = {'load': Field(default='A')}
|
||||
save = Field(default='A')
|
||||
|
||||
class B(A): pass
|
||||
class B(A):
|
||||
pass
|
||||
|
||||
class C(Item):
|
||||
fields = {'load': Field(default='C')}
|
||||
save = Field(default='C')
|
||||
|
||||
class D(B, C): pass
|
||||
class D(B, C):
|
||||
pass
|
||||
|
||||
item = D(save='X', load='Y')
|
||||
self.assertEqual(item['save'], 'X')
|
||||
|
|
@ -164,7 +166,8 @@ class ItemTest(unittest.TestCase):
|
|||
'save': {'default': 'A'}})
|
||||
|
||||
# D class inverted
|
||||
class E(C, B): pass
|
||||
class E(C, B):
|
||||
pass
|
||||
|
||||
self.assertEqual(E(save='X')['save'], 'X')
|
||||
self.assertEqual(E(load='X')['load'], 'X')
|
||||
|
|
@ -177,7 +180,8 @@ class ItemTest(unittest.TestCase):
|
|||
save = Field(default='A')
|
||||
load = Field(default='A')
|
||||
|
||||
class B(A): pass
|
||||
class B(A):
|
||||
pass
|
||||
|
||||
class C(A):
|
||||
fields = {'update': Field(default='C')}
|
||||
|
|
@ -206,14 +210,16 @@ class ItemTest(unittest.TestCase):
|
|||
fields = {'load': Field(default='A')}
|
||||
save = Field(default='A')
|
||||
|
||||
class B(A): pass
|
||||
class B(A):
|
||||
pass
|
||||
|
||||
class C(object):
|
||||
fields = {'load': Field(default='C')}
|
||||
not_allowed = Field(default='not_allowed')
|
||||
save = Field(default='C')
|
||||
|
||||
class D(B, C): pass
|
||||
class D(B, C):
|
||||
pass
|
||||
|
||||
self.assertRaises(KeyError, D, not_allowed='value')
|
||||
self.assertEqual(D(save='X')['save'], 'X')
|
||||
|
|
@ -221,7 +227,8 @@ class ItemTest(unittest.TestCase):
|
|||
'load': {'default': 'A'}})
|
||||
|
||||
# D class inverted
|
||||
class E(C, B): pass
|
||||
class E(C, B):
|
||||
pass
|
||||
|
||||
self.assertRaises(KeyError, E, not_allowed='value')
|
||||
self.assertEqual(E(save='X')['save'], 'X')
|
||||
|
|
@ -259,6 +266,7 @@ class ItemTest(unittest.TestCase):
|
|||
with catch_warnings(record=True) as warnings:
|
||||
item = Item()
|
||||
self.assertEqual(len(warnings), 0)
|
||||
|
||||
class SubclassedItem(Item):
|
||||
pass
|
||||
subclassed_item = SubclassedItem()
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@ import re
|
|||
import unittest
|
||||
from warnings import catch_warnings
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.http import HtmlResponse, XmlResponse
|
||||
from scrapy.link import Link
|
||||
|
|
@ -16,7 +14,6 @@ from tests import get_testdata
|
|||
class Base:
|
||||
class LinkExtractorTestCase(unittest.TestCase):
|
||||
extractor_cls = None
|
||||
escapes_whitespace = False
|
||||
|
||||
def setUp(self):
|
||||
body = get_testdata('link_extractor', 'linkextractor.html')
|
||||
|
|
@ -30,10 +27,7 @@ class Base:
|
|||
|
||||
def test_extract_all_links(self):
|
||||
lx = self.extractor_cls()
|
||||
if self.escapes_whitespace:
|
||||
page4_url = 'http://example.com/page%204.html'
|
||||
else:
|
||||
page4_url = 'http://example.com/page 4.html'
|
||||
page4_url = 'http://example.com/page%204.html'
|
||||
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
|
|
@ -214,7 +208,7 @@ class Base:
|
|||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
|
||||
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
|
||||
self.assertEqual(links,
|
||||
[Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
|
||||
[Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
|
||||
|
||||
def test_restrict_xpaths_concat_in_handle_data(self):
|
||||
"""html entities cause SGMLParser to call handle_data hook twice"""
|
||||
|
|
@ -310,10 +304,7 @@ class Base:
|
|||
|
||||
def test_attrs(self):
|
||||
lx = self.extractor_cls(attrs="href")
|
||||
if self.escapes_whitespace:
|
||||
page4_url = 'http://example.com/page%204.html'
|
||||
else:
|
||||
page4_url = 'http://example.com/page 4.html'
|
||||
page4_url = 'http://example.com/page%204.html'
|
||||
|
||||
self.assertEqual(lx.extract_links(self.response), [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
|
|
@ -506,7 +497,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
|
|||
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
|
||||
])
|
||||
|
||||
@pytest.mark.xfail
|
||||
def test_restrict_xpaths_with_html_entities(self):
|
||||
super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()
|
||||
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import unittest
|
|||
|
||||
from testfixtures import LogCapture
|
||||
from twisted.internet import defer
|
||||
from twisted.python.failure import Failure
|
||||
from twisted.trial.unittest import TestCase as TwistedTestCase
|
||||
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
|
|
@ -62,15 +63,46 @@ class LogFormatterTestCase(unittest.TestCase):
|
|||
assert all(isinstance(x, str) for x in lines)
|
||||
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
|
||||
|
||||
def test_error(self):
|
||||
def test_item_error(self):
|
||||
# In practice, the complete traceback is shown by passing the
|
||||
# 'exc_info' argument to the logging function
|
||||
item = {'key': 'value'}
|
||||
exception = Exception()
|
||||
response = Response("http://www.example.com")
|
||||
logkws = self.formatter.error(item, exception, response, self.spider)
|
||||
logkws = self.formatter.item_error(item, exception, response, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
self.assertEqual(logline, u"'Error processing {'key': 'value'}'")
|
||||
self.assertEqual(logline, u"Error processing {'key': 'value'}")
|
||||
|
||||
def test_spider_error(self):
|
||||
# In practice, the complete traceback is shown by passing the
|
||||
# 'exc_info' argument to the logging function
|
||||
failure = Failure(Exception())
|
||||
request = Request("http://www.example.com", headers={'Referer': 'http://example.org'})
|
||||
response = Response("http://www.example.com", request=request)
|
||||
logkws = self.formatter.spider_error(failure, request, response, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
self.assertEqual(
|
||||
logline,
|
||||
"Spider error processing <GET http://www.example.com> (referer: http://example.org)"
|
||||
)
|
||||
|
||||
def test_download_error_short(self):
|
||||
# In practice, the complete traceback is shown by passing the
|
||||
# 'exc_info' argument to the logging function
|
||||
failure = Failure(Exception())
|
||||
request = Request("http://www.example.com")
|
||||
logkws = self.formatter.download_error(failure, request, self.spider)
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
self.assertEqual(logline, "Error downloading <GET http://www.example.com>")
|
||||
|
||||
def test_download_error_long(self):
|
||||
# In practice, the complete traceback is shown by passing the
|
||||
# 'exc_info' argument to the logging function
|
||||
failure = Failure(Exception())
|
||||
request = Request("http://www.example.com")
|
||||
logkws = self.formatter.download_error(failure, request, self.spider, "Some message")
|
||||
logline = logkws['msg'] % logkws['args']
|
||||
self.assertEqual(logline, "Error downloading <GET http://www.example.com>: Some message")
|
||||
|
||||
def test_scraped(self):
|
||||
item = CustomItem()
|
||||
|
|
|
|||
|
|
@ -121,5 +121,6 @@ class MailSenderTest(unittest.TestCase):
|
|||
self.assertEqual(text.get_charset(), Charset('utf-8'))
|
||||
self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -26,10 +26,9 @@ class MediaDownloadSpider(SimpleSpider):
|
|||
self.media_key: [],
|
||||
self.media_urls_key: [
|
||||
self._process_url(response.urljoin(href))
|
||||
for href in response.xpath('''
|
||||
//table[thead/tr/th="Filename"]
|
||||
/tbody//a/@href
|
||||
''').getall()],
|
||||
for href in response.xpath(
|
||||
'//table[thead/tr/th="Filename"]/tbody//a/@href'
|
||||
).getall()],
|
||||
}
|
||||
yield item
|
||||
|
||||
|
|
@ -99,8 +98,9 @@ class FileDownloadCrawlTestCase(TestCase):
|
|||
if self.expected_checksums is not None:
|
||||
checksums = set(
|
||||
i['checksum']
|
||||
for item in items
|
||||
for i in item[self.media_key])
|
||||
for item in items
|
||||
for i in item[self.media_key]
|
||||
)
|
||||
self.assertEqual(checksums, self.expected_checksums)
|
||||
|
||||
# check that the image files where actually written to the media store
|
||||
|
|
|
|||
|
|
@ -272,7 +272,7 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
prefix = pipeline_cls.__name__.upper()
|
||||
settings = self._generate_fake_settings(prefix=prefix)
|
||||
user_pipeline = pipeline_cls.from_settings(Settings(settings))
|
||||
for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
|
||||
for pipe_cls_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
|
||||
custom_value = settings.get(prefix + "_" + settings_attr)
|
||||
self.assertNotEqual(custom_value, self.default_cls_settings[pipe_cls_attr])
|
||||
self.assertEqual(getattr(user_pipeline, pipe_inst_attr), custom_value)
|
||||
|
|
@ -286,7 +286,6 @@ class FilesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
self.assertEqual(pipeline.files_result_field, "this")
|
||||
self.assertEqual(pipeline.files_urls_field, "that")
|
||||
|
||||
|
||||
def test_user_defined_subclass_default_key_names(self):
|
||||
"""Test situation when user defines subclass of FilesPipeline,
|
||||
but uses attribute names for default pipeline (without prefixing
|
||||
|
|
@ -360,7 +359,7 @@ class TestGCSFilesStore(unittest.TestCase):
|
|||
self.assertIn('checksum', s)
|
||||
self.assertEqual(s['checksum'], 'zc2oVgXkbQr2EQdSdw3OPA==')
|
||||
u = urlparse(uri)
|
||||
content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:]+path)
|
||||
content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path)
|
||||
self.assertEqual(content, data)
|
||||
self.assertEqual(blob.metadata, {'foo': 'bar'})
|
||||
self.assertEqual(blob.cache_control, GCSFilesStore.CACHE_CONTROL)
|
||||
|
|
|
|||
|
|
@ -177,7 +177,6 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
|
|||
IMAGES_RESULT_FIELD='images'
|
||||
)
|
||||
|
||||
|
||||
def setUp(self):
|
||||
self.tempdir = mkdtemp()
|
||||
|
||||
|
|
|
|||
|
|
@ -304,6 +304,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
|
|||
return response
|
||||
|
||||
rsp1 = Response('http://url')
|
||||
|
||||
def rsp1_func():
|
||||
dfd = Deferred().addCallback(_check_downloading)
|
||||
reactor.callLater(.1, dfd.callback, rsp1)
|
||||
|
|
|
|||
|
|
@ -90,5 +90,6 @@ class ResponseTypesTest(unittest.TestCase):
|
|||
# check that mime.types files shipped with scrapy are loaded
|
||||
self.assertEqual(responsetypes.mimetypes.guess_type('x.scrapytest')[0], 'x-scrapy/test')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
from pytest import mark
|
||||
from twisted.internet import defer
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy import signals, Request, Spider
|
||||
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
|
||||
|
||||
from tests.mockserver import MockServer
|
||||
|
||||
|
||||
class ItemSpider(Spider):
|
||||
name = 'itemspider'
|
||||
|
||||
def start_requests(self):
|
||||
for index in range(10):
|
||||
yield Request(self.mockserver.url('/status?n=200&id=%d' % index),
|
||||
meta={'index': index})
|
||||
|
||||
def parse(self, response):
|
||||
return {'index': response.meta['index']}
|
||||
|
||||
|
||||
class AsyncSignalTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.mockserver = MockServer()
|
||||
self.mockserver.__enter__()
|
||||
self.items = []
|
||||
|
||||
def tearDown(self):
|
||||
self.mockserver.__exit__(None, None, None)
|
||||
|
||||
async def _on_item_scraped(self, item):
|
||||
item = await get_from_asyncio_queue(item)
|
||||
self.items.append(item)
|
||||
|
||||
@mark.only_asyncio()
|
||||
@defer.inlineCallbacks
|
||||
def test_simple_pipeline(self):
|
||||
crawler = get_crawler(ItemSpider)
|
||||
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
|
||||
yield crawler.crawl(mockserver=self.mockserver)
|
||||
self.assertEqual(len(self.items), 10)
|
||||
for index in range(10):
|
||||
self.assertIn({'index': index}, self.items)
|
||||
|
|
@ -94,7 +94,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
|
|||
|
||||
class RaiseExceptionProcessSpiderOutputMiddleware:
|
||||
def process_spider_output(self, response, result, spider):
|
||||
1/0
|
||||
1 / 0
|
||||
|
||||
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
|
||||
self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ class NotGeneratorCallbackSpider(Spider):
|
|||
yield Request(self.mockserver.url('/status?n=200'))
|
||||
|
||||
def parse(self, response):
|
||||
return [{'test': 1}, {'test': 1/0}]
|
||||
return [{'test': 1}, {'test': 1 / 0}]
|
||||
|
||||
|
||||
# ================================================================================
|
||||
|
|
|
|||
|
|
@ -1,7 +1,12 @@
|
|||
import pickle
|
||||
|
||||
from queuelib.tests import test_queue as t
|
||||
from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFifoDiskQueue, PickleLifoDiskQueue
|
||||
from scrapy.squeues import (
|
||||
MarshalFifoDiskQueueNonRequest as MarshalFifoDiskQueue,
|
||||
MarshalLifoDiskQueueNonRequest as MarshalLifoDiskQueue,
|
||||
PickleFifoDiskQueueNonRequest as PickleFifoDiskQueue,
|
||||
PickleLifoDiskQueueNonRequest as PickleLifoDiskQueue
|
||||
)
|
||||
from scrapy.item import Item, Field
|
||||
from scrapy.http import Request
|
||||
from scrapy.loader import ItemLoader
|
||||
|
|
@ -31,7 +36,9 @@ def nonserializable_object_test(self):
|
|||
self.assertRaises(ValueError, q.push, lambda x: x)
|
||||
else:
|
||||
# Use a different unpickleable object
|
||||
class A(object): pass
|
||||
class A(object):
|
||||
pass
|
||||
|
||||
a = A()
|
||||
a.__reduce__ = a.__reduce_ex__ = None
|
||||
self.assertRaises(ValueError, q.push, a)
|
||||
|
|
|
|||
|
|
@ -83,7 +83,6 @@ class BuildComponentListTest(unittest.TestCase):
|
|||
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
|
||||
|
||||
|
||||
|
||||
class UtilsConfTestCase(unittest.TestCase):
|
||||
|
||||
def test_arglist_to_dict(self):
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from scrapy.utils.defer import mustbe_deferred, process_chain, \
|
|||
class MustbeDeferredTest(unittest.TestCase):
|
||||
def test_success_function(self):
|
||||
steps = []
|
||||
|
||||
def _append(v):
|
||||
steps.append(v)
|
||||
return steps
|
||||
|
|
@ -20,6 +21,7 @@ class MustbeDeferredTest(unittest.TestCase):
|
|||
|
||||
def test_unfired_deferred(self):
|
||||
steps = []
|
||||
|
||||
def _append(v):
|
||||
steps.append(v)
|
||||
dfd = defer.Deferred()
|
||||
|
|
@ -102,7 +104,7 @@ class IterErrbackTest(unittest.TestCase):
|
|||
def iterbad():
|
||||
for x in range(10):
|
||||
if x == 5:
|
||||
a = 1/0
|
||||
a = 1 / 0
|
||||
yield x
|
||||
|
||||
errors = []
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
|
|||
# ignore subclassing warnings
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('ignore', ScrapyDeprecationWarning)
|
||||
|
||||
class UserClass(Deprecated):
|
||||
pass
|
||||
|
||||
|
|
@ -233,6 +234,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
|
|||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
AlsoDeprecated()
|
||||
|
||||
class UserClass(AlsoDeprecated):
|
||||
pass
|
||||
|
||||
|
|
@ -247,6 +249,7 @@ class WarnWhenSubclassedTest(unittest.TestCase):
|
|||
with mock.patch('inspect.stack', side_effect=IndexError):
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
DeprecatedName = create_deprecated_class('DeprecatedName', NewName)
|
||||
|
||||
class SubClass(DeprecatedName):
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -387,7 +387,6 @@ class TestHelper(unittest.TestCase):
|
|||
self.assertTrue(type(r1) is type(r2))
|
||||
self.assertTrue(type(r1) is not type(r3))
|
||||
|
||||
|
||||
def _assert_type_and_value(self, a, b, obj):
|
||||
self.assertTrue(type(a) is type(b),
|
||||
'Got {}, expected {} for {!r}'.format(type(a), type(b), obj))
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ class FailureToExcInfoTest(unittest.TestCase):
|
|||
|
||||
def test_failure(self):
|
||||
try:
|
||||
0/0
|
||||
0 / 0
|
||||
except ZeroDivisionError:
|
||||
exc_info = sys.exc_info()
|
||||
failure = Failure()
|
||||
|
|
|
|||
|
|
@ -104,7 +104,6 @@ class BinaryIsTextTest(unittest.TestCase):
|
|||
assert not binary_is_text(b"\x02\xa3")
|
||||
|
||||
|
||||
|
||||
class UtilsPythonTestCase(unittest.TestCase):
|
||||
|
||||
def test_equal_attributes(self):
|
||||
|
|
@ -154,7 +153,9 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
self.assertFalse(equal_attributes(a, b, [compare_z, 'x']))
|
||||
|
||||
def test_weakkeycache(self):
|
||||
class _Weakme(object): pass
|
||||
class _Weakme(object):
|
||||
pass
|
||||
|
||||
_values = count()
|
||||
wk = WeakKeyCache(lambda k: next(_values))
|
||||
k = _Weakme()
|
||||
|
|
@ -215,7 +216,6 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
self.assertEqual(
|
||||
get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
|
||||
|
||||
|
||||
def test_without_none_values(self):
|
||||
self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
|
||||
self.assertEqual(without_none_values((1, None, 3, 4)), (1, 3, 4))
|
||||
|
|
@ -223,5 +223,6 @@ class UtilsPythonTestCase(unittest.TestCase):
|
|||
without_none_values({'one': 1, 'none': None, 'three': 3, 'four': 4}),
|
||||
{'one': 1, 'three': 3, 'four': 4})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -83,5 +83,6 @@ class UtilsRequestTest(unittest.TestCase):
|
|||
request_httprepr(Request("file:///tmp/foo.txt"))
|
||||
request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
import asyncio
|
||||
|
||||
from pytest import mark
|
||||
from testfixtures import LogCapture
|
||||
from twisted.trial import unittest
|
||||
from twisted.python.failure import Failure
|
||||
|
|
@ -5,6 +8,7 @@ from twisted.internet import defer, reactor
|
|||
from pydispatch import dispatcher
|
||||
|
||||
from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
|
||||
from scrapy.utils.test import get_from_asyncio_queue
|
||||
|
||||
|
||||
class SendCatchLogTest(unittest.TestCase):
|
||||
|
|
@ -40,7 +44,7 @@ class SendCatchLogTest(unittest.TestCase):
|
|||
|
||||
def error_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.error_handler)
|
||||
a = 1/0
|
||||
a = 1 / 0
|
||||
|
||||
def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
|
|
@ -54,7 +58,7 @@ class SendCatchLogDeferredTest(SendCatchLogTest):
|
|||
return send_catch_log_deferred(signal, *a, **kw)
|
||||
|
||||
|
||||
class SendCatchLogDeferredTest2(SendCatchLogTest):
|
||||
class SendCatchLogDeferredTest2(SendCatchLogDeferredTest):
|
||||
|
||||
def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
|
|
@ -63,8 +67,24 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
|
|||
reactor.callLater(0, d.callback, "OK")
|
||||
return d
|
||||
|
||||
def _get_result(self, signal, *a, **kw):
|
||||
return send_catch_log_deferred(signal, *a, **kw)
|
||||
|
||||
class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
|
||||
|
||||
async def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
assert arg == 'test'
|
||||
await defer.succeed(42)
|
||||
return "OK"
|
||||
|
||||
|
||||
@mark.only_asyncio()
|
||||
class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
|
||||
|
||||
async def ok_handler(self, arg, handlers_called):
|
||||
handlers_called.add(self.ok_handler)
|
||||
assert arg == 'test'
|
||||
await asyncio.sleep(0.2)
|
||||
return await get_from_asyncio_queue("OK")
|
||||
|
||||
|
||||
class SendCatchLogTest2(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -38,5 +38,6 @@ class UtilsRenderTemplateFileTestCase(unittest.TestCase):
|
|||
os.remove(render_path)
|
||||
assert not os.path.exists(render_path) # Failure of test iself
|
||||
|
||||
|
||||
if '__main__' == __name__:
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
|
||||
url = 'javascript:%20document.orderform_2581_1190810811.mode.value=%27add%27;%20javascript:%20document.orderform_2581_1190810811.submit%28%29'
|
||||
self.assertFalse(url_is_from_any_domain(url, ['testdomain.com']))
|
||||
self.assertFalse(url_is_from_any_domain(url+'.testdomain.com', ['testdomain.com']))
|
||||
self.assertFalse(url_is_from_any_domain(url + '.testdomain.com', ['testdomain.com']))
|
||||
|
||||
def test_url_is_from_spider(self):
|
||||
spider = Spider(name='example.com')
|
||||
|
|
@ -201,7 +201,8 @@ def create_skipped_scheme_t(args):
|
|||
assert url.startswith(args[1])
|
||||
return do_expected
|
||||
|
||||
for k, args in enumerate ([
|
||||
|
||||
for k, args in enumerate([
|
||||
('/index', 'file://'),
|
||||
('/index.html', 'file://'),
|
||||
('./index.html', 'file://'),
|
||||
|
|
@ -229,7 +230,7 @@ for k, args in enumerate ([
|
|||
], start=1):
|
||||
t_method = create_guess_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_%03d' % k
|
||||
setattr (GuessSchemeTest, t_method.__name__, t_method)
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
# TODO: the following tests do not pass with current implementation
|
||||
for k, args in enumerate([
|
||||
|
|
@ -238,7 +239,7 @@ for k, args in enumerate([
|
|||
], start=1):
|
||||
t_method = create_skipped_scheme_t(args)
|
||||
t_method.__name__ = 'test_uri_skipped_%03d' % k
|
||||
setattr (GuessSchemeTest, t_method.__name__, t_method)
|
||||
setattr(GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
|
||||
class StripUrl(unittest.TestCase):
|
||||
|
|
|
|||
|
|
@ -51,22 +51,22 @@ class ParseUrlTestCase(unittest.TestCase):
|
|||
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip + ':100', lip, 100, '/foo?c=v&c2=v2')),
|
||||
|
||||
("http://127.0.0.1", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/", ('http', lip, lip, 80, '/')),
|
||||
("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')),
|
||||
("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')),
|
||||
("http://127.0.0.1:12345/foo", ('http', lip+':12345', lip, 12345, '/foo')),
|
||||
("http://127.0.0.1:12345/foo", ('http', lip + ':12345', lip, 12345, '/foo')),
|
||||
("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')),
|
||||
("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')),
|
||||
|
||||
("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')),
|
||||
("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')),
|
||||
("https://127.0.0.1:12345/", ('https', lip+':12345', lip, 12345, '/')),
|
||||
("https://127.0.0.1:12345/", ('https', lip + ':12345', lip, 12345, '/')),
|
||||
|
||||
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')),
|
||||
("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')),
|
||||
|
|
@ -294,6 +294,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
finished = self.assertFailure(
|
||||
getPage(self.getURL("wait"), timeout=0.000001),
|
||||
defer.TimeoutError)
|
||||
|
||||
def cleanup(passthrough):
|
||||
# Clean up the server which is hanging around not doing
|
||||
# anything.
|
||||
|
|
|
|||
Loading…
Reference in New Issue