RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into pypy

This commit is contained in:
Adrián Chaves 2020-05-06 13:47:09 +02:00 committed by GitHub
parent f36189957e 17c0cf64ae
commit de1ffb88f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 689 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.bumpversion.cfg
View File

@ -1,5 +1,5 @@
[bumpversion]
current_version = 2.0.0
current_version = 2.1.0
commit = True
tag = True
tag_name = {new_version}

24
.travis.yml
View File

@ -11,26 +11,24 @@ matrix:
python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=py35
python: 3.5
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
- python: 3.5
- env: TOXENV=pinned
python: 3.5
- env: TOXENV=py35-asyncio
- env: TOXENV=asyncio
python: 3.5.2
- env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0
- env: TOXENV=py36
python: 3.6
- python: 3.6
- env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1
- env: TOXENV=py37
python: 3.7
- env: TOXENV=py38
- python: 3.7
- env: PYPI_RELEASE_JOB=true
python: 3.8
- env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=py38-asyncio
- env: TOXENV=asyncio
python: 3.8
- env: TOXENV=docs
python: 3.7 # Keep in sync with .readthedocs.yml
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
@ -63,4 +61,4 @@ deploy:
on:
tags: true
repo: scrapy/scrapy
condition: "$TOXENV == py37 && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"
condition: "$PYPI_RELEASE_JOB == true && $TRAVIS_TAG =~ ^[0-9]+[.][0-9]+[.][0-9]+(rc[0-9]+|[.]dev[0-9]+)?$"

2
conftest.py
View File

@ -12,6 +12,8 @@ collect_ignore = [
"scrapy/utils/testsite.py",
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
# Py36-only parts of respective tests
*_py_files("tests/py36"),
]

2
docs/conf.py
View File

@ -295,8 +295,6 @@ intersphinx_mapping = {
# ------------------------------------
hoverxref_auto_ref = True
hoverxref_project = "scrapy"
hoverxref_version = release
hoverxref_role_types = {
"class": "tooltip",
"confval": "tooltip",

147
docs/news.rst
View File

@ -3,6 +3,153 @@
Release notes
=============
.. _release-2.1.0:
Scrapy 2.1.0 (2020-04-24)
-------------------------
Highlights:
* New :setting:`FEEDS` setting to export to multiple feeds
* New :attr:`Response.ip_address <scrapy.http.Response.ip_address>` attribute
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* :exc:`AssertionError` exceptions triggered by :ref:`assert <assert>`
statements have been replaced by new exception types, to support running
Python in optimized mode (see :option:`-O`) without changing Scrapy’s
behavior in any unexpected ways.
If you catch an :exc:`AssertionError` exception from Scrapy, update your
code to catch the corresponding new exception.
(:issue:`4440`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
* The ``LOG_UNSERIALIZABLE_REQUESTS`` setting is no longer supported, use
:setting:`SCHEDULER_DEBUG` instead (:issue:`4385`)
* The ``REDIRECT_MAX_METAREFRESH_DELAY`` setting is no longer supported, use
:setting:`METAREFRESH_MAXDELAY` instead (:issue:`4385`)
* The :class:`~scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware`
middleware has been removed, including the entire
:class:`scrapy.downloadermiddlewares.chunked` module; chunked transfers
work out of the box (:issue:`4431`)
* The ``spiders`` property has been removed from
:class:`~scrapy.crawler.Crawler`, use :class:`CrawlerRunner.spider_loader
<scrapy.crawler.CrawlerRunner.spider_loader>` or instantiate
:setting:`SPIDER_LOADER_CLASS` with your settings instead (:issue:`4398`)
* The ``MultiValueDict``, ``MultiValueDictKeyError``, and ``SiteNode``
classes have been removed from :mod:`scrapy.utils.datatypes`
(:issue:`4400`)
Deprecations
~~~~~~~~~~~~
* The ``FEED_FORMAT`` and ``FEED_URI`` settings have been deprecated in
favor of the new :setting:`FEEDS` setting (:issue:`1336`, :issue:`3858`,
:issue:`4507`)
New features
~~~~~~~~~~~~
* A new setting, :setting:`FEEDS`, allows configuring multiple output feeds
with different settings each (:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The :command:`crawl` and :command:`runspider` commands now support multiple
``-o`` parameters (:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The :command:`crawl` and :command:`runspider` commands now support
specifying an output format by appending ``:<format>`` to the output file
(:issue:`1336`, :issue:`3858`, :issue:`4507`)
* The new :attr:`Response.ip_address <scrapy.http.Response.ip_address>`
attribute gives access to the IP address that originated a response
(:issue:`3903`, :issue:`3940`)
* A warning is now issued when a value in
:attr:`~scrapy.spiders.Spider.allowed_domains` includes a port
(:issue:`50`, :issue:`3198`, :issue:`4413`)
* Zsh completion now excludes used option aliases from the completion list
(:issue:`4438`)
Bug fixes
~~~~~~~~~
* :ref:`Request serialization <request-serialization>` no longer breaks for
callbacks that are spider attributes which are assigned a function with a
different name (:issue:`4500`)
* ``None`` values in :attr:`~scrapy.spiders.Spider.allowed_domains` no longer
cause a :exc:`TypeError` exception (:issue:`4410`)
* Zsh completion no longer allows options after arguments (:issue:`4438`)
* zope.interface 5.0.0 and later versions are now supported
(:issue:`4447`, :issue:`4448`)
* :meth:`Spider.make_requests_from_url
<scrapy.spiders.Spider.make_requests_from_url>`, deprecated in Scrapy
1.4.0, now issues a warning when used (:issue:`4412`)
Documentation
~~~~~~~~~~~~~
* Improved the documentation about signals that allow their handlers to
return a :class:`~twisted.internet.defer.Deferred` (:issue:`4295`,
:issue:`4390`)
* Our PyPI entry now includes links for our documentation, our source code
repository and our issue tracker (:issue:`4456`)
* Covered the `curl2scrapy <https://michael-shub.github.io/curl2scrapy/>`_
service in the documentation (:issue:`4206`, :issue:`4455`)
* Removed references to the Guppy library, which only works in Python 2
(:issue:`4285`, :issue:`4343`)
* Extended use of InterSphinx to link to Python 3 documentation
(:issue:`4444`, :issue:`4445`)
* Added support for Sphinx 3.0 and later (:issue:`4475`, :issue:`4480`,
:issue:`4496`, :issue:`4503`)
Quality assurance
~~~~~~~~~~~~~~~~~
* Removed warnings about using old, removed settings (:issue:`4404`)
* Removed a warning about importing
:class:`~twisted.internet.testing.StringTransport` from
``twisted.test.proto_helpers`` in Twisted 19.7.0 or newer (:issue:`4409`)
* Removed outdated Debian package build files (:issue:`4384`)
* Removed :class:`object` usage as a base class (:issue:`4430`)
* Removed code that added support for old versions of Twisted that we no
longer support (:issue:`4472`)
* Fixed code style issues (:issue:`4468`, :issue:`4469`, :issue:`4471`,
:issue:`4481`)
* Removed :func:`twisted.internet.defer.returnValue` calls (:issue:`4443`,
:issue:`4446`, :issue:`4489`)
.. _release-2.0.1:
Scrapy 2.0.1 (2020-03-18)

4
docs/topics/coroutines.rst
View File

@ -7,10 +7,6 @@ Coroutines
Scrapy has :ref:`partial support <coroutine-support>` for the
:ref:`coroutine syntax <async>`.
.. warning:: :mod:`asyncio` support in Scrapy is experimental. Future Scrapy
versions may introduce related API and behavior changes without a
deprecation period or warning.
.. _coroutine-support:
Supported callables

5
docs/topics/practices.rst
View File

@ -35,8 +35,9 @@ Here's an example showing how to run a single spider with it.
...
process = CrawlerProcess(settings={
'FEED_FORMAT': 'json',
'FEED_URI': 'items.json'
"FEEDS": {
"items.json": {"format": "json"},
},
})
process.crawl(MySpider)

18
docs/topics/request-response.rst
View File

@ -36,7 +36,7 @@ Request objects
:type url: string
:param callback: the function that will be called with the response of this
request (once its downloaded) as its first parameter. For more information
request (once it's downloaded) as its first parameter. For more information
see :ref:`topics-request-response-ref-request-callback-arguments` below.
If a Request doesn't specify a callback, the spider's
:meth:`~scrapy.spiders.Spider.parse` method will be used.
@ -616,6 +616,12 @@ Response objects
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
.. versionadded:: 2.1.0
The ``ip_address`` parameter.
.. attribute:: Response.url
A string containing the URL of the response.
@ -705,6 +711,16 @@ Response objects
Only populated for ``https`` responses, ``None`` otherwise.
.. attribute:: Response.ip_address
.. versionadded:: 2.1.0
The IP address of the server from which the Response originated.
This attribute is currently only populated by the HTTP 1.1 download
handler, i.e. for ``http(s)`` responses. For other handlers,
:attr:`ip_address` is always ``None``.
.. method:: Response.copy()
Returns a new Response which is a copy of this Response.

16
docs/topics/settings.rst
View File

@ -420,10 +420,9 @@ connections (for ``HTTP10DownloadHandler``).
.. note::
HTTP/1.0 is rarely used nowadays so you can safely ignore this setting,
unless you use Twisted<11.1, or if you really want to use HTTP/1.0
and override :setting:`DOWNLOAD_HANDLERS_BASE` for ``http(s)`` scheme
accordingly, i.e. to
``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
unless you really want to use HTTP/1.0 and override
:setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
.. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY
@ -447,7 +446,6 @@ or even enable client-side authentication (and various other things).
Scrapy also has another context factory class that you can set,
``'scrapy.core.downloader.contextfactory.BrowserLikeContextFactory'``,
which uses the platform's certificates to validate remote endpoints.
**This is only available if you use Twisted>=14.0.**
If you do use a custom ContextFactory, make sure its ``__init__`` method
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
@ -494,10 +492,6 @@ This setting must be one of these string values:
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
.. note::
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
or above (Twisted>=14.0 if you can).
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -660,8 +654,6 @@ If you want to disable it set to 0.
spider attribute and per-request using :reqmeta:`download_maxsize`
Request.meta key.
This feature needs Twisted >= 11.1.
.. setting:: DOWNLOAD_WARNSIZE
DOWNLOAD_WARNSIZE
@ -679,8 +671,6 @@ If you want to disable it set to 0.
spider attribute and per-request using :reqmeta:`download_warnsize`
Request.meta key.
This feature needs Twisted >= 11.1.
.. setting:: DOWNLOAD_FAIL_ON_DATALOSS
DOWNLOAD_FAIL_ON_DATALOSS

11
docs/topics/shell.rst
View File

@ -156,6 +156,17 @@ First, we launch the shell::
scrapy shell 'https://scrapy.org' --nolog
.. note::
Remember to always enclose URLs in quotes when running the Scrapy shell from
the command line, otherwise URLs containing arguments (i.e. the ``&`` character)
will not work.
On Windows, use double quotes instead::
scrapy shell "https://scrapy.org" --nolog
Then, the shell fetches the URL (using the Scrapy downloader) and prints the
list of available objects and useful shortcuts (you'll notice that these lines
all start with the ``[s]`` prefix)::

96
docs/topics/signals.rst
View File

@ -16,8 +16,7 @@ deliver the arguments that the handler receives.
You can connect to signals (or send your own) through the
:ref:`topics-api-signals`.
Here is a simple example showing how you can catch signals and perform some action:
::
Here is a simple example showing how you can catch signals and perform some action::
from scrapy import signals
from scrapy import Spider
@ -52,9 +51,45 @@ Deferred signal handlers
========================
Some signals support returning :class:`~twisted.internet.defer.Deferred`
objects from their handlers, see the :ref:`topics-signals-ref` below to know
which ones.
objects from their handlers, allowing you to run asynchronous code that
does not block Scrapy. If a signal handler returns a
:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
:class:`~twisted.internet.defer.Deferred` to fire.
Let's take an example::
class SignalSpider(scrapy.Spider):
name = 'signals'
start_urls = ['http://quotes.toscrape.com/page/1/']
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
return spider
def item_scraped(self, item):
# Send the scraped item to the server
d = treq.post(
'http://example.com/post',
json.dumps(item).encode('ascii'),
headers={b'Content-Type': [b'application/json']}
)
# The next item will be scraped only after
# deferred (d) is fired
return d
def parse(self, response):
for quote in response.css('div.quote'):
yield {
'text': quote.css('span.text::text').get(),
'author': quote.css('small.author::text').get(),
'tags': quote.css('div.tags a.tag::text').getall(),
}
See the :ref:`topics-signals-ref` below to know which signals support
:class:`~twisted.internet.defer.Deferred`.
.. _topics-signals-ref:
@ -66,9 +101,12 @@ Built-in signals reference
Here's the list of Scrapy built-in signals and their meaning.
engine_started
Engine signals
--------------
engine_started
~~~~~~~~~~~~~~
.. signal:: engine_started
.. function:: engine_started()
@ -81,7 +119,7 @@ engine_started
getting fired before :signal:`spider_opened`.
engine_stopped
--------------
~~~~~~~~~~~~~~
.. signal:: engine_stopped
.. function:: engine_stopped()
@ -91,9 +129,20 @@ engine_stopped
This signal supports returning deferreds from their handlers.
item_scraped
Item signals
------------
.. note::
As at max :setting:`CONCURRENT_ITEMS` items are processed in
parallel, many deferreds are fired together using
:class:`~twisted.internet.defer.DeferredList`. Hence the next
batch waits for the :class:`~twisted.internet.defer.DeferredList`
to fire and then runs the respective item signal handler for
the next batch of scraped items.
item_scraped
~~~~~~~~~~~~
.. signal:: item_scraped
.. function:: item_scraped(item, response, spider)
@ -112,7 +161,7 @@ item_scraped
:type response: :class:`~scrapy.http.Response` object
item_dropped
------------
~~~~~~~~~~~~
.. signal:: item_dropped
.. function:: item_dropped(item, response, exception, spider)
@ -137,7 +186,7 @@ item_dropped
:type exception: :exc:`~scrapy.exceptions.DropItem` exception
item_error
------------
~~~~~~~~~~
.. signal:: item_error
.. function:: item_error(item, response, spider, failure)
@ -159,8 +208,11 @@ item_error
:param failure: the exception raised
:type failure: twisted.python.failure.Failure
Spider signals
--------------
spider_closed
-------------
~~~~~~~~~~~~~
.. signal:: spider_closed
.. function:: spider_closed(spider, reason)
@ -183,7 +235,7 @@ spider_closed
:type reason: str
spider_opened
-------------
~~~~~~~~~~~~~
.. signal:: spider_opened
.. function:: spider_opened(spider)
@ -198,7 +250,7 @@ spider_opened
:type spider: :class:`~scrapy.spiders.Spider` object
spider_idle
-----------
~~~~~~~~~~~
.. signal:: spider_idle
.. function:: spider_idle(spider)
@ -228,7 +280,7 @@ spider_idle
due to duplication).
spider_error
------------
~~~~~~~~~~~~
.. signal:: spider_error
.. function:: spider_error(failure, response, spider)
@ -246,8 +298,11 @@ spider_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
Request signals
---------------
request_scheduled
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: request_scheduled
.. function:: request_scheduled(request, spider)
@ -264,7 +319,7 @@ request_scheduled
:type spider: :class:`~scrapy.spiders.Spider` object
request_dropped
---------------
~~~~~~~~~~~~~~~
.. signal:: request_dropped
.. function:: request_dropped(request, spider)
@ -281,7 +336,7 @@ request_dropped
:type spider: :class:`~scrapy.spiders.Spider` object
request_reached_downloader
---------------------------
~~~~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_reached_downloader
.. function:: request_reached_downloader(request, spider)
@ -297,7 +352,7 @@ request_reached_downloader
:type spider: :class:`~scrapy.spiders.Spider` object
request_left_downloader
-----------------------
~~~~~~~~~~~~~~~~~~~~~~~
.. signal:: request_left_downloader
.. function:: request_left_downloader(request, spider)
@ -315,8 +370,11 @@ request_left_downloader
:param spider: the spider that yielded the request
:type spider: :class:`~scrapy.spiders.Spider` object
Response signals
----------------
response_received
-----------------
~~~~~~~~~~~~~~~~~
.. signal:: response_received
.. function:: response_received(response, request, spider)
@ -336,7 +394,7 @@ response_received
:type spider: :class:`~scrapy.spiders.Spider` object
response_downloaded
-------------------
~~~~~~~~~~~~~~~~~~~
.. signal:: response_downloaded
.. function:: response_downloaded(response, request, spider)

38
pytest.ini
View File

@ -35,27 +35,27 @@ flake8-ignore =
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/fetch.py E401 E501 E128
scrapy/commands/genspider.py E128 E501
scrapy/commands/parse.py E128 E501 E731
scrapy/commands/parse.py E128 E501
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501
scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
scrapy/contracts/__init__.py E501
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E501 E128 E127
scrapy/core/scheduler.py E501
scrapy/core/scraper.py E501 E128 W504
scrapy/core/spidermw.py E501 E731 E126
scrapy/core/scraper.py E501 E128
scrapy/core/spidermw.py E501 E126
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501
scrapy/core/downloader/tls.py E501
scrapy/core/downloader/webclient.py E731 E501 E128 E126
scrapy/core/downloader/webclient.py E501 E128 E126
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E128 E127
scrapy/core/downloader/handlers/http10.py E501
@ -68,7 +68,7 @@ flake8-ignore =
scrapy/downloadermiddlewares/httpcache.py E501 E126
scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/redirect.py E501
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
@ -79,7 +79,7 @@ flake8-ignore =
scrapy/extensions/httpcache.py E128 E501
scrapy/extensions/memdebug.py E501
scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/telnet.py E501
scrapy/extensions/throttle.py E501
# scrapy/http
scrapy/http/common.py E501
@ -90,8 +90,8 @@ flake8-ignore =
scrapy/http/response/__init__.py E501 E128
scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
scrapy/linkextractors/__init__.py E731 E501 E402 W504
scrapy/linkextractors/lxmlhtml.py E501 E731
scrapy/linkextractors/__init__.py E501 E402
scrapy/linkextractors/lxmlhtml.py E501
# scrapy/loader
scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
@ -110,7 +110,7 @@ flake8-ignore =
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
scrapy/spidermiddlewares/referer.py E501 E129 W504
scrapy/spidermiddlewares/referer.py E501 E129
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
@ -125,7 +125,7 @@ flake8-ignore =
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127
scrapy/utils/gz.py E501 W504
scrapy/utils/gz.py E501
scrapy/utils/http.py F403
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501
@ -184,7 +184,7 @@ flake8-ignore =
tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E126
tests/test_downloadermiddleware_cookies.py E741 E501 E128 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
@ -197,7 +197,7 @@ flake8-ignore =
tests/test_downloadermiddleware_stats.py E501
tests/test_dupefilters.py E501 E741 E128 E124
tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E128 E124
tests/test_exporters.py E501 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
@ -207,14 +207,14 @@ flake8-ignore =
tests/test_item.py E128 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
tests/test_loader.py E501 E731 E741 E128 E117
tests/test_loader.py E501 E741 E128 E117
tests/test_logformatter.py E128 E501 E122
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128 E126
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E731 E128
tests/test_pipeline_media.py E501 E741 E128
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
@ -234,14 +234,14 @@ flake8-ignore =
tests/test_utils_datatypes.py E402 E501
tests/test_utils_defer.py E501 F841
tests/test_utils_deprecate.py F841 E501
tests/test_utils_http.py E501 E128 W504
tests/test_utils_http.py E501 E128
tests/test_utils_iterators.py E501 E128 E129
tests/test_utils_log.py E741
tests/test_utils_python.py E501 E731
tests/test_utils_python.py E501
tests/test_utils_reqser.py E501 E128
tests/test_utils_request.py E501 E128
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731
tests/test_utils_signal.py E741 F841
tests/test_utils_sitemap.py E128 E501 E124
tests/test_utils_url.py E501 E127 E125 E501 E126 E123
tests/test_webclient.py E501 E128 E122 E402 E123 E126

2
scrapy/VERSION
View File

@ -1 +1 @@
2.0.0
2.1.0

3
scrapy/commands/__init__.py
View File

@ -23,7 +23,8 @@ class ScrapyCommand:
self.settings = None # set in scrapy.cmdline
def set_crawler(self, crawler):
assert not hasattr(self, '_crawler'), "crawler already set"
if hasattr(self, '_crawler'):
raise RuntimeError("crawler already set")
self._crawler = crawler
def syntax(self):

4
scrapy/commands/fetch.py
View File

@ -49,8 +49,8 @@ class Command(ScrapyCommand):
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
raise UsageError()
cb = lambda x: self._print_response(x, opts)
request = Request(args[0], callback=cb, dont_filter=True)
request = Request(args[0], callback=self._print_response,
cb_kwargs={"opts": opts}, dont_filter=True)
# by default, let the framework handle redirects,
# i.e. command handles all codes expect 3xx
if not opts.no_redirect:

5
scrapy/commands/parse.py
View File

@ -146,9 +146,8 @@ class Command(ScrapyCommand):
if not self.spidercls:
logger.error('Unable to find spider for: %(url)s', {'url': url})
# Request requires callback argument as callable or None, not string
request = Request(url, None)
_start_requests = lambda s: [self.prepare_request(s, request, opts)]
def _start_requests(spider):
yield self.prepare_request(spider, Request(url), opts)
self.spidercls.start_requests = _start_requests
def start_parsing(self, url, opts):

4
scrapy/contracts/__init__.py
View File

@ -17,10 +17,10 @@ class ContractsManager:
self.contracts[contract.name] = contract
def tested_methods_from_spidercls(self, spidercls):
is_method = re.compile(r"^\s*@", re.MULTILINE).search
methods = []
for key, value in getmembers(spidercls):
if (callable(value) and value.__doc__ and
re.search(r'^\s*@', value.__doc__, re.MULTILINE)):
if callable(value) and value.__doc__ and is_method(value.__doc__):
methods.append(key)
return methods

6
scrapy/contracts/default.py
View File

@ -58,7 +58,11 @@ class ReturnsContract(Contract):
def __init__(self, *args, **kwargs):
super(ReturnsContract, self).__init__(*args, **kwargs)
assert len(self.args) in [1, 2, 3]
if len(self.args) not in [1, 2, 3]:
raise ValueError(
"Incorrect argument quantity: expected 1, 2 or 3, got %i"
% len(self.args)
)
self.obj_name = self.args[0] or None
self.obj_type = self.objects[self.obj_name]

2
scrapy/core/downloader/__init__.py
View File

@ -173,7 +173,7 @@ class Downloader:
return response
dfd.addCallback(_downloaded)
# 3. After response arrives, remove the request from transferring
# 3. After response arrives, remove the request from transferring
# state to free up the transferring slot so it can be used by the
# following requests (perhaps those which came from the downloader
# middleware itself)

54
scrapy/core/downloader/handlers/http11.py
View File

@ -1,5 +1,6 @@
"""Download handlers for http and https schemes"""
import ipaddress
import logging
import re
import warnings
@ -373,7 +374,13 @@ class ScrapyAgent:
def _cb_bodyready(self, txresponse, request):
# deliverBody hangs for responses without body
if txresponse.length == 0:
return txresponse, b'', None, None
return {
"txresponse": txresponse,
"body": b"",
"flags": None,
"certificate": None,
"ip_address": None,
}
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
@ -409,12 +416,17 @@ class ScrapyAgent:
return d
def _cb_bodydone(self, result, request, url):
txresponse, body, flags, certificate = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
return respcls(url=url, status=status, headers=headers, body=body,
flags=flags, certificate=certificate)
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
return respcls(
url=url,
status=int(result["txresponse"].code),
headers=headers,
body=result["body"],
flags=result["flags"],
certificate=result["certificate"],
ip_address=result["ip_address"],
)
@implementer(IBodyProducer)
@ -449,12 +461,16 @@ class _ResponseReader(protocol.Protocol):
self._reached_warnsize = False
self._bytes_received = 0
self._certificate = None
self._ip_address = None
def connectionMade(self):
if self._certificate is None:
with suppress(AttributeError):
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
if self._ip_address is None:
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
def dataReceived(self, bodyBytes):
# This maybe called several times after cancel was called with buffered data.
if self._finished.called:
@ -486,16 +502,34 @@ class _ResponseReader(protocol.Protocol):
body = self._bodybuf.getvalue()
if reason.check(ResponseDone):
self._finished.callback((self._txresponse, body, None, self._certificate))
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": None,
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
if reason.check(PotentialDataLoss):
self._finished.callback((self._txresponse, body, ['partial'], self._certificate))
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["partial"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if not self._fail_on_dataloss:
self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate))
self._finished.callback({
"txresponse": self._txresponse,
"body": body,
"flags": ["dataloss"],
"certificate": self._certificate,
"ip_address": self._ip_address,
})
return
elif not self._fail_on_dataloss_warned:

5
scrapy/core/downloader/middleware.py
View File

@ -45,8 +45,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
@defer.inlineCallbacks
def process_response(response):
assert response is not None, 'Received None in process_response'
if isinstance(response, Request):
if response is None:
raise TypeError("Received None in process_response")
elif isinstance(response, Request):
return response
for method in self.methods['process_response']:

9
scrapy/core/downloader/webclient.py
View File

@ -14,13 +14,12 @@ from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
# Assume parsed is urlparse-d from Request.url,
# which was passed via safe_url_string and is ascii-only.
b = lambda s: to_bytes(s, encoding='ascii')
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
path = b(path)
host = b(parsed.hostname)
path = to_bytes(path, encoding="ascii")
host = to_bytes(parsed.hostname, encoding="ascii")
port = parsed.port
scheme = b(parsed.scheme)
netloc = b(parsed.netloc)
scheme = to_bytes(parsed.scheme, encoding="ascii")
netloc = to_bytes(parsed.netloc, encoding="ascii")
if port is None:
port = 443 if scheme == b'https' else 80
return scheme, netloc, host, port, path

26
scrapy/core/engine.py
View File

@ -73,7 +73,8 @@ class ExecutionEngine:
@defer.inlineCallbacks
def start(self):
"""Start the execution engine"""
assert not self.running, "Engine already running"
if self.running:
raise RuntimeError("Engine already running")
self.start_time = time()
yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
self.running = True
@ -82,7 +83,8 @@ class ExecutionEngine:
def stop(self):
"""Stop the execution engine gracefully"""
assert self.running, "Engine not running"
if not self.running:
raise RuntimeError("Engine not running")
self.running = False
dfd = self._close_all_spiders()
return dfd.addBoth(lambda _: self._finish_stopping_engine())
@ -165,7 +167,11 @@ class ExecutionEngine:
return d
def _handle_downloader_output(self, response, request, spider):
assert isinstance(response, (Request, Response, Failure)), response
if not isinstance(response, (Request, Response, Failure)):
raise TypeError(
"Incorrect type: expected Request, Response or Failure, got %s: %r"
% (type(response), response)
)
# downloader middleware can return requests (for example, redirects)
if isinstance(response, Request):
self.crawl(response, spider)
@ -205,8 +211,8 @@ class ExecutionEngine:
return not bool(self.slot)
def crawl(self, request, spider):
assert spider in self.open_spiders, \
"Spider %r not opened when crawling: %s" % (spider.name, request)
if spider not in self.open_spiders:
raise RuntimeError("Spider %r not opened when crawling: %s" % (spider.name, request))
self.schedule(request, spider)
self.slot.nextcall.schedule()
@ -232,7 +238,11 @@ class ExecutionEngine:
slot.add_request(request)
def _on_success(response):
assert isinstance(response, (Response, Request))
if not isinstance(response, (Response, Request)):
raise TypeError(
"Incorrect type: expected Response or Request, got %s: %r"
% (type(response), response)
)
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
@ -253,8 +263,8 @@ class ExecutionEngine:
@defer.inlineCallbacks
def open_spider(self, spider, start_requests=(), close_if_idle=True):
assert self.has_capacity(), "No free spider slot when opening %r" % \
spider.name
if not self.has_capacity():
raise RuntimeError("No free spider slot when opening %r" % spider.name)
logger.info("Spider opened", extra={'spider': spider})
nextcall = CallLaterOnce(self._next_request, spider)
scheduler = self.scheduler_cls.from_crawler(self.crawler)

6
scrapy/core/scraper.py
View File

@ -123,7 +123,11 @@ class Scraper:
def _scrape(self, response, request, spider):
"""Handle the downloaded response or failure through the spider
callback/errback"""
assert isinstance(response, (Response, Failure))
if not isinstance(response, (Response, Failure)):
raise TypeError(
"Incorrect type: expected Response or Failure, got %s: %r"
% (type(response), response)
)
dfd = self._scrape2(response, request, spider) # returns spider's processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)

3
scrapy/crawler.py
View File

@ -78,7 +78,8 @@ class Crawler:
@defer.inlineCallbacks
def crawl(self, *args, **kwargs):
assert not self.crawling, "Crawling already taking place"
if self.crawling:
raise RuntimeError("Crawling already taking place")
self.crawling = True
try:

11
scrapy/downloadermiddlewares/redirect.py
View File

@ -60,11 +60,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
Handle redirection of requests based on response status
and meta-refresh html tag.
"""
def process_response(self, request, response, spider):
if (request.meta.get('dont_redirect', False) or
response.status in getattr(spider, 'handle_httpstatus_list', []) or
response.status in request.meta.get('handle_httpstatus_list', []) or
request.meta.get('handle_httpstatus_all', False)):
if (
request.meta.get('dont_redirect', False)
or response.status in getattr(spider, 'handle_httpstatus_list', [])
or response.status in request.meta.get('handle_httpstatus_list', [])
or request.meta.get('handle_httpstatus_all', False)
):
return response
allowed_status = (301, 302, 303, 307, 308)

6
scrapy/extensions/telnet.py
View File

@ -76,8 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
"""An implementation of IPortal"""
@defers
def login(self_, credentials, mind, *interfaces):
if not (credentials.username == self.username.encode('utf8') and
credentials.checkPassword(self.password.encode('utf8'))):
if not (
credentials.username == self.username.encode('utf8')
and credentials.checkPassword(self.password.encode('utf8'))
):
raise ValueError("Invalid credentials")
protocol = telnet.TelnetBootstrapProtocol(

3
scrapy/http/request/__init__.py
View File

@ -24,7 +24,8 @@ class Request(object_ref):
self.method = str(method).upper()
self._set_url(url)
self._set_body(body)
assert isinstance(priority, int), "Request priority not an integer: %r" % priority
if not isinstance(priority, int):
raise TypeError("Request priority not an integer: %r" % priority)
self.priority = priority
if callback is not None and not callable(callback):

7
scrapy/http/response/__init__.py
View File

@ -17,7 +17,8 @@ from scrapy.utils.trackref import object_ref
class Response(object_ref):
def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None):
def __init__(self, url, status=200, headers=None, body=b'', flags=None,
request=None, certificate=None, ip_address=None):
self.headers = Headers(headers or {})
self.status = int(status)
self._set_body(body)
@ -25,6 +26,7 @@ class Response(object_ref):
self.request = request
self.flags = [] if flags is None else list(flags)
self.certificate = certificate
self.ip_address = ip_address
@property
def cb_kwargs(self):
@ -87,7 +89,8 @@ class Response(object_ref):
"""Create a new Response with the same attributes except for those
given new values.
"""
for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']:
for x in ['url', 'status', 'headers', 'body',
'request', 'flags', 'certificate', 'ip_address']:
kwargs.setdefault(x, getattr(self, x))
cls = kwargs.pop('cls', self.__class__)
return cls(*args, **kwargs)

13
scrapy/linkextractors/__init__.py
View File

@ -45,8 +45,14 @@ IGNORED_EXTENSIONS = [
_re_type = type(re.compile("", 0))
_matches = lambda url, regexs: any(r.search(url) for r in regexs)
_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
def _matches(url, regexs):
return any(r.search(url) for r in regexs)
def _is_valid_url(url):
return url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'}
class FilteringLinkExtractor:
@ -55,8 +61,7 @@ class FilteringLinkExtractor:
def __new__(cls, *args, **kwargs):
from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
if (issubclass(cls, FilteringLinkExtractor) and
not issubclass(cls, LxmlLinkExtractor)):
if issubclass(cls, FilteringLinkExtractor) and not issubclass(cls, LxmlLinkExtractor):
warn('scrapy.linkextractors.FilteringLinkExtractor is deprecated, '
'please use scrapy.linkextractors.LinkExtractor instead',
ScrapyDeprecationWarning, stacklevel=2)

6
scrapy/linkextractors/lxmlhtml.py
View File

@ -98,11 +98,9 @@ class LxmlLinkExtractor(FilteringLinkExtractor):
unique=True, process_value=None, deny_extensions=None, restrict_css=(),
strip=True, restrict_text=None):
tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
tag_func = lambda x: x in tags
attr_func = lambda x: x in attrs
lx = LxmlParserLinkExtractor(
tag=tag_func,
attr=attr_func,
tag=lambda x: x in tags,
attr=lambda x: x in attrs,
unique=unique,
process=process_value,
strip=strip,

6
scrapy/pipelines/files.py
View File

@ -106,7 +106,8 @@ class S3FilesStore:
else:
from boto.s3.connection import S3Connection
self.S3Connection = S3Connection
assert uri.startswith('s3://')
if not uri.startswith("s3://"):
raise ValueError("Incorrect URI scheme in %s, expected 's3'" % uri)
self.bucket, self.prefix = uri[5:].split('/', 1)
def stat_file(self, path, info):
@ -266,7 +267,8 @@ class FTPFilesStore:
USE_ACTIVE_MODE = None
def __init__(self, uri):
assert uri.startswith('ftp://')
if not uri.startswith("ftp://"):
raise ValueError("Incorrect URI scheme in %s, expected 'ftp'" % uri)
u = urlparse(uri)
self.port = u.port
self.host = u.hostname

14
scrapy/spidermiddlewares/referer.py
View File

@ -163,9 +163,10 @@ class StrictOriginPolicy(ReferrerPolicy):
name = POLICY_STRICT_ORIGIN
def referrer(self, response_url, request_url):
if ((self.tls_protected(response_url) and
self.potentially_trustworthy(request_url))
or not self.tls_protected(response_url)):
if (
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)
@ -213,9 +214,10 @@ class StrictOriginWhenCrossOriginPolicy(ReferrerPolicy):
origin = self.origin(response_url)
if origin == self.origin(request_url):
return self.stripped_referrer(response_url)
elif ((self.tls_protected(response_url) and
self.potentially_trustworthy(request_url))
or not self.tls_protected(response_url)):
elif (
self.tls_protected(response_url) and self.potentially_trustworthy(request_url)
or not self.tls_protected(response_url)
):
return self.origin_referrer(response_url)

3
scrapy/utils/gz.py
View File

@ -52,8 +52,7 @@ def is_gzipped(response):
"""Return True if the response is gzipped, or False otherwise"""
ctype = response.headers.get('Content-Type', b'')
cenc = response.headers.get('Content-Encoding', b'').lower()
return (_is_gzipped(ctype) or
(_is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')))
return _is_gzipped(ctype) or _is_octetstream(ctype) and cenc in (b'gzip', b'x-gzip')
def gzip_magic_number(response):

10
scrapy/utils/iterators.py
View File

@ -128,10 +128,12 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
type(obj).__name__)
if not isinstance(obj, expected_types):
expected_types_str = " or ".join(t.__name__ for t in expected_types)
raise TypeError(
"Object %r must be %s, not %s"
% (obj, expected_types_str, type(obj).__name__)
)
if isinstance(obj, Response):
if not unicode:
return obj.body

3
scrapy/utils/reactor.py
View File

@ -9,7 +9,8 @@ from scrapy.utils.misc import load_object
def listen_tcp(portrange, host, factory):
"""Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor
assert len(portrange) <= 2, "invalid portrange: %s" % portrange
if len(portrange) > 2:
raise ValueError("invalid portrange: %s" % portrange)
if not portrange:
return reactor.listenTCP(0, factory, interface=host)
if not hasattr(portrange, '__iter__'):

31
scrapy/utils/reqser.py
View File

@ -1,6 +1,8 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
import inspect
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@ -68,20 +70,6 @@ def request_from_dict(d, spider=None):
)
def _is_private_method(name):
return name.startswith('__') and not name.endswith('__')
def _mangle_private_name(obj, func, name):
qualname = getattr(func, '__qualname__', None)
if qualname is None:
classname = obj.__class__.__name__.lstrip('_')
return '_%s%s' % (classname, name)
else:
splits = qualname.split('.')
return '_%s%s' % (splits[-2], splits[-1])
def _find_method(obj, func):
if obj:
try:
@ -90,10 +78,17 @@ def _find_method(obj, func):
pass
else:
if func_self is obj:
name = func.__func__.__name__
if _is_private_method(name):
return _mangle_private_name(obj, func, name)
return name
members = inspect.getmembers(obj, predicate=inspect.ismethod)
for name, obj_func in members:
# We need to use __func__ to access the original
# function object because instance method objects
# are generated each time attribute is retrieved from
# instance.
#
# Reference: The standard type hierarchy
# https://docs.python.org/3/reference/datamodel.html
if obj_func.__func__ is func.__func__:
return name
raise ValueError("Function %s is not a method of: %s" % (func, obj))

38
tests/CrawlerRunner/ip_address.py Normal file
View File

@ -0,0 +1,38 @@
from urllib.parse import urlparse
from twisted.internet import reactor
from twisted.names.client import createResolver
from scrapy import Spider, Request
from scrapy.crawler import CrawlerRunner
from scrapy.utils.log import configure_logging
from tests.mockserver import MockServer, MockDNSServer
class LocalhostSpider(Spider):
name = "localhost_spider"
def start_requests(self):
yield Request(self.url)
def parse(self, response):
netloc = urlparse(response.url).netloc
self.logger.info("Host: %s" % netloc.split(":")[0])
self.logger.info("Type: %s" % type(response.ip_address))
self.logger.info("IP address: %s" % response.ip_address)
if __name__ == "__main__":
with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
port = urlparse(mock_http_server.http_address).port
url = "http://not.a.real.domain:{port}/echo".format(port=port)
servers = [(mock_dns_server.host, mock_dns_server.port)]
reactor.installResolver(createResolver(servers=servers))
configure_logging()
runner = CrawlerRunner()
d = runner.crawl(LocalhostSpider, url=url)
d.addBoth(lambda _: reactor.stop())
reactor.run()

99
tests/mockserver.py
View File

@ -1,3 +1,4 @@
import argparse
import json
import os
import random
@ -6,18 +7,19 @@ from subprocess import Popen, PIPE
from urllib.parse import urlencode
from OpenSSL import SSL
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.internet import defer, reactor, ssl
from twisted.internet.task import deferLater
from twisted.names import dns, error
from twisted.names.server import DNSServerFactory
from twisted.web.resource import EncodingResourceWrapper, Resource
from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
from twisted.web.static import File
from twisted.web.test.test_webclient import PayloadResource
from twisted.web.server import GzipEncoderFactory
from twisted.web.resource import EncodingResourceWrapper
from twisted.web.util import redirectTo
from twisted.internet import reactor, ssl
from twisted.internet.task import deferLater
from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
from scrapy.utils.test import get_testenv
def getarg(request, name, default=None, type=None):
@ -198,12 +200,10 @@ class Root(Resource):
return b'Scrapy mock HTTP server\n'
class MockServer():
class MockServer:
def __enter__(self):
from scrapy.utils.test import get_testenv
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'],
stdout=PIPE, env=get_testenv())
http_address = self.proc.stdout.readline().strip().decode('ascii')
https_address = self.proc.stdout.readline().strip().decode('ascii')
@ -224,11 +224,45 @@ class MockServer():
return host + path
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
def _resolve(self, name):
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
return [answer], [], []
def query(self, query, timeout=None):
if query.type == dns.A:
return defer.succeed(self._resolve(query.name.name))
return defer.fail(error.DomainError())
def lookupAllRecords(self, name, timeout=None):
return defer.succeed(self._resolve(name))
class MockDNSServer:
def __enter__(self):
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
stdout=PIPE, env=get_testenv())
host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
self.host = host
self.port = int(port)
return self
def __exit__(self, exc_type, exc_value, traceback):
self.proc.kill()
self.proc.communicate()
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
factory = ssl.DefaultOpenSSLContextFactory(
os.path.join(os.path.dirname(__file__), keyfile),
os.path.join(os.path.dirname(__file__), certfile),
)
os.path.join(os.path.dirname(__file__), keyfile),
os.path.join(os.path.dirname(__file__), certfile),
)
if cipher_string:
ctx = factory.getContext()
# disabling TLS1.2+ because it unconditionally enables some strong ciphers
@ -238,19 +272,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
if __name__ == "__main__":
root = Root()
factory = Site(root)
httpPort = reactor.listenTCP(0, factory)
contextFactory = ssl_context_factory()
httpsPort = reactor.listenSSL(0, factory, contextFactory)
parser = argparse.ArgumentParser()
parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http")
args = parser.parse_args()
def print_listening():
httpHost = httpPort.getHost()
httpsHost = httpsPort.getHost()
httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
print(httpAddress)
print(httpsAddress)
if args.type == "http":
root = Root()
factory = Site(root)
httpPort = reactor.listenTCP(0, factory)
contextFactory = ssl_context_factory()
httpsPort = reactor.listenSSL(0, factory, contextFactory)
def print_listening():
httpHost = httpPort.getHost()
httpsHost = httpsPort.getHost()
httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port)
httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port)
print(httpAddress)
print(httpsAddress)
elif args.type == "dns":
clients = [MockDNSResolver()]
factory = DNSServerFactory(clients=clients)
protocol = dns.DNSDatagramProtocol(controller=factory)
listener = reactor.listenUDP(0, protocol)
def print_listening():
host = listener.getHost()
print("%s:%s" % (host.host, host.port))
reactor.callWhenRunning(print_listening)
reactor.run()

21
tests/test_crawl.py
View File

@ -1,6 +1,9 @@
import json
import logging
import sys
from ipaddress import IPv4Address
from socket import gethostbyname
from urllib.parse import urlparse
from pytest import mark
from testfixtures import LogCapture
@ -436,3 +439,21 @@ with multiples lines
self.assertIsInstance(cert, Certificate)
self.assertEqual(cert.getSubject().commonName, b"localhost")
self.assertEqual(cert.getIssuer().commonName, b"localhost")
@defer.inlineCallbacks
def test_dns_server_ip_address_none(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/status?n=200')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
self.assertIsNone(ip_address)
@defer.inlineCallbacks
def test_dns_server_ip_address(self):
crawler = self.runner.create_crawler(SingleRequestSpider)
url = self.mockserver.url('/echo?body=test')
expected_netloc, _ = urlparse(url).netloc.split(':')
yield crawler.crawl(seed=url, mockserver=self.mockserver)
ip_address = crawler.spider.meta['responses'][0].ip_address
self.assertIsInstance(ip_address, IPv4Address)
self.assertEqual(str(ip_address), gethostbyname(expected_netloc))

19
tests/test_crawler.py
View File

@ -274,9 +274,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
class CrawlerProcessSubprocess(unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
class ScriptRunnerMixin:
def run_script(self, script_name):
script_path = os.path.join(self.script_dir, script_name)
args = (sys.executable, script_path)
@ -285,6 +283,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
stdout, stderr = p.communicate()
return stderr.decode('utf-8')
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
def test_simple(self):
log = self.run_script('simple.py')
self.assertIn('Spider closed (finished)', log)
@ -332,3 +334,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
log = self.run_script("twisted_reactor_asyncio.py")
self.assertIn("Spider closed (finished)", log)
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
def test_response_ip_address(self):
log = self.run_script("ip_address.py")
self.assertIn("INFO: Spider closed (finished)", log)
self.assertIn("INFO: Host: not.a.real.domain", log)
self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
self.assertIn("INFO: IP address: 127.0.0.1", log)

7
tests/test_downloadermiddleware_cookies.py
View File

@ -13,10 +13,9 @@ from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
class CookiesMiddlewareTest(TestCase):
def assertCookieValEqual(self, first, second, msg=None):
cookievaleq = lambda cv: re.split(r';\s*', cv.decode('latin1'))
return self.assertEqual(
sorted(cookievaleq(first)),
sorted(cookievaleq(second)), msg)
def split_cookies(cookies):
return sorted(re.split(r";\s*", cookies.decode("latin1")))
return self.assertEqual(split_cookies(first), split_cookies(second), msg=msg)
def setUp(self):
self.spider = Spider('foo')

11
tests/test_exporters.py
View File

@ -215,11 +215,12 @@ class CsvItemExporterTest(BaseItemExporterTest):
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
first = to_unicode(first)
second = to_unicode(second)
csvsplit = lambda csv: [sorted(re.split(r'(,|\s+)', line))
for line in csv.splitlines(True)]
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
def split_csv(csv):
return [
sorted(re.split(r"(,|\s+)", line))
for line in to_unicode(csv).splitlines(True)
]
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')

18
tests/test_pipeline_media.py
View File

@ -198,12 +198,19 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
pipeline_class = MockedMediaPipeline
def _callback(self, result):
self.pipe._mockcalled.append('request_callback')
return result
def _errback(self, result):
self.pipe._mockcalled.append('request_errback')
return result
@inlineCallbacks
def test_result_succeed(self):
cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
rsp = Response('http://url1')
req = Request('http://url1', meta=dict(response=rsp), callback=cb, errback=eb)
req = Request('http://url1', meta=dict(response=rsp),
callback=self._callback, errback=self._errback)
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, rsp)])
@ -214,10 +221,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
@inlineCallbacks
def test_result_failure(self):
self.pipe.LOG_FAILED_RESULTS = False
cb = lambda _: self.pipe._mockcalled.append('request_callback') or _
eb = lambda _: self.pipe._mockcalled.append('request_errback') or _
fail = Failure(Exception())
req = Request('http://url1', meta=dict(response=fail), callback=cb, errback=eb)
req = Request('http://url1', meta=dict(response=fail),
callback=self._callback, errback=self._errback)
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(False, fail)])

8
tests/test_utils_http.py
View File

@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
chunked_body += "8\r\n" + "sequence\r\n"
chunked_body += "0\r\n\r\n"
body = decode_chunked_transfer(chunked_body)
self.assertEqual(body,
"This is the data in the first chunk\r\n" +
"and this is the second one\r\n" +
"consequence")
self.assertEqual(
body,
"This is the data in the first chunk\r\nand this is the second one\r\nconsequence"
)

2
tests/test_utils_iterators.py
View File

@ -157,7 +157,7 @@ class XmliterTestCase(unittest.TestCase):
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, 'product')
self.assertRaises(AssertionError, next, i)
self.assertRaises(TypeError, next, i)
def test_xmliter_encoding(self):
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'

4
tests/test_utils_python.py
View File

@ -145,7 +145,9 @@ class UtilsPythonTestCase(unittest.TestCase):
get_z = operator.itemgetter('z')
get_meta = operator.attrgetter('meta')
compare_z = lambda obj: get_z(get_meta(obj))
def compare_z(obj):
return get_z(get_meta(obj))
self.assertTrue(equal_attributes(a, b, [compare_z, 'x']))
# fail z equality

92
tests/test_utils_reqser.py
View File

@ -2,7 +2,7 @@ import unittest
from scrapy.http import Request, FormRequest
from scrapy.spiders import Spider
from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name
from scrapy.utils.reqser import request_to_dict, request_from_dict
class RequestSerializationTest(unittest.TestCase):
@ -69,6 +69,26 @@ class RequestSerializationTest(unittest.TestCase):
errback=self.spider.handle_error)
self._assert_serializes_ok(r, spider=self.spider)
def test_reference_callback_serialization(self):
r = Request("http://www.example.com",
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
self.assertEqual(request_dict['callback'], 'parse_item_reference')
self.assertEqual(request_dict['errback'], 'handle_error_reference')
def test_private_reference_callback_serialization(self):
r = Request("http://www.example.com",
callback=self.spider._TestSpider__parse_item_reference,
errback=self.spider._TestSpider__handle_error_reference)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = request_to_dict(r, self.spider)
self.assertEqual(request_dict['callback'],
'_TestSpider__parse_item_reference')
self.assertEqual(request_dict['errback'],
'_TestSpider__handle_error_reference')
def test_private_callback_serialization(self):
r = Request("http://www.example.com",
callback=self.spider._TestSpider__parse_item_private,
@ -81,41 +101,6 @@ class RequestSerializationTest(unittest.TestCase):
errback=self.spider.handle_error)
self._assert_serializes_ok(r, spider=self.spider)
def test_private_callback_name_matching(self):
self.assertTrue(_is_private_method('__a'))
self.assertTrue(_is_private_method('__a_'))
self.assertTrue(_is_private_method('__a_a'))
self.assertTrue(_is_private_method('__a_a_'))
self.assertTrue(_is_private_method('__a__a'))
self.assertTrue(_is_private_method('__a__a_'))
self.assertTrue(_is_private_method('__a___a'))
self.assertTrue(_is_private_method('__a___a_'))
self.assertTrue(_is_private_method('___a'))
self.assertTrue(_is_private_method('___a_'))
self.assertTrue(_is_private_method('___a_a'))
self.assertTrue(_is_private_method('___a_a_'))
self.assertTrue(_is_private_method('____a_a_'))
self.assertFalse(_is_private_method('_a'))
self.assertFalse(_is_private_method('_a_'))
self.assertFalse(_is_private_method('__a__'))
self.assertFalse(_is_private_method('__'))
self.assertFalse(_is_private_method('___'))
self.assertFalse(_is_private_method('____'))
def _assert_mangles_to(self, obj, name):
func = getattr(obj, name)
self.assertEqual(
_mangle_private_name(obj, func, func.__name__),
name
)
def test_private_name_mangling(self):
self._assert_mangles_to(
self.spider, '_TestSpider__parse_item_private')
self._assert_mangles_to(
self.spider, '_TestSpiderMixin__mixin_callback')
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
self.assertRaises(ValueError, request_to_dict, r)
@ -125,14 +110,49 @@ class RequestSerializationTest(unittest.TestCase):
r = Request("http://www.example.com", callback=self.spider.parse_item)
self.assertRaises(ValueError, request_to_dict, r)
def test_unserializable_callback3(self):
"""Parser method is removed or replaced dynamically."""
class MySpider(Spider):
name = 'my_spider'
def parse(self, response):
pass
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
setattr(spider, 'parse', None)
self.assertRaises(ValueError, request_to_dict, r, spider=spider)
class TestSpiderMixin:
def __mixin_callback(self, response):
pass
def parse_item(response):
pass
def handle_error(failure):
pass
def private_parse_item(response):
pass
def private_handle_error(failure):
pass
class TestSpider(Spider, TestSpiderMixin):
name = 'test'
parse_item_reference = parse_item
handle_error_reference = handle_error
__parse_item_reference = private_parse_item
__handle_error_reference = private_handle_error
def parse_item(self, response):
pass

4
tests/test_utils_signal.py
View File

@ -90,8 +90,10 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
class SendCatchLogTest2(unittest.TestCase):
def test_error_logged_if_deferred_not_supported(self):
def test_handler():
return defer.Deferred()
test_signal = object()
test_handler = lambda: defer.Deferred()
dispatcher.connect(test_handler, test_signal)
with LogCapture() as l:
send_catch_log(test_signal)

18
tox.ini
View File

@ -74,11 +74,15 @@ deps =
changedir = docs
deps =
-rdocs/requirements.txt
setenv =
READTHEDOCS_PROJECT=scrapy
READTHEDOCS_VERSION=master
[testenv:docs]
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b html . {envtmpdir}/html
@ -86,6 +90,7 @@ commands =
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -b coverage . {envtmpdir}/coverage
@ -93,19 +98,10 @@ commands =
basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
[asyncio]
[testenv:asyncio]
commands =
{[testenv]commands} --reactor=asyncio
[testenv:py35-asyncio]
basepython = python3.5
deps = {[testenv]deps}
commands = {[asyncio]commands}
[testenv:py38-asyncio]
basepython = python3.8
deps = {[testenv]deps}
commands = {[asyncio]commands}