diff --git a/docs/conf.py b/docs/conf.py index eab366efd..914d1d05f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -275,6 +275,7 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'coverage': ('https://coverage.readthedocs.io/en/stable', None), 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), diff --git a/docs/contributing.rst b/docs/contributing.rst index 81bb50a77..234c4bcee 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -243,14 +243,13 @@ the Python 3.6 :doc:`tox ` environment using all your CPU cores:: tox -e py36 -- scrapy tests -n auto -To see coverage report install `coverage`_ (``pip install coverage``) and run: +To see coverage report install :doc:`coverage ` +(``pip install coverage``) and run: ``coverage report`` see output of ``coverage --help`` for more options like html or xml report. -.. _coverage: https://pypi.python.org/pypi/coverage - Writing tests ------------- diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 81c8dab03..de3f38023 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -477,6 +477,8 @@ ItemLoader objects .. attribute:: item The :class:`~scrapy.item.Item` object being parsed by this Item Loader. + This is mostly used as a property so when attempting to override this + value, you may want to check out :attr:`default_item_class` first. .. attribute:: context diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 41252d73e..6c7eb737b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,4 +1,5 @@ import logging +import pprint import signal import warnings @@ -45,7 +46,8 @@ class Crawler(object): logging.root.addHandler(handler) d = dict(overridden_settings(self.settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) + logger.info("Overridden settings:\n%(settings)s", + {'settings': pprint.pformat(d)}) if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 9deba40d6..d8dabdf13 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,4 +1,3 @@ -import os import logging from collections import defaultdict diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 495b103d1..4e06f8236 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,11 +1,19 @@ from email.utils import formatdate + from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed + from scrapy import signals -from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object