diff --git a/docs/conf.py b/docs/conf.py index 04472cf86..eab366efd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -275,8 +275,10 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), 'sphinx': ('https://www.sphinx-doc.org/en/master', None), + 'tox': ('https://tox.readthedocs.io/en/latest', None), 'twisted': ('https://twistedmatrix.com/documents/current', None), } diff --git a/docs/contributing.rst b/docs/contributing.rst index 68ae2bf3c..81bb50a77 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -196,14 +196,14 @@ Tests Tests are implemented using the :doc:`Twisted unit-testing framework `. Running tests requires -`tox`_. +:doc:`tox `. .. _running-tests: Running tests ------------- -Make sure you have a recent enough `tox`_ installation: +Make sure you have a recent enough :doc:`tox ` installation: ``tox --version`` @@ -219,26 +219,27 @@ To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` -To run the tests on a specific tox_ environment, use ``-e `` with an -environment name from ``tox.ini``. For example, to run the tests with Python -3.6 use:: +To run the tests on a specific :doc:`tox ` environment, use +``-e `` with an environment name from ``tox.ini``. For example, to run +the tests with Python 3.6 use:: tox -e py36 -You can also specify a comma-separated list of environmets, and use `tox’s -parallel mode`_ to run the tests on multiple environments in parallel:: +You can also specify a comma-separated list of environmets, and use :ref:`tox’s +parallel mode ` to run the tests on multiple environments in +parallel:: tox -e py27,py36 -p auto -To pass command-line options to pytest_, add them after ``--`` in your call to -tox_. Using ``--`` overrides the default positional arguments defined in -``tox.ini``, so you must include those default positional arguments -(``scrapy tests``) after ``--`` as well:: +To pass command-line options to :doc:`pytest `, add them after +``--`` in your call to :doc:`tox `. Using ``--`` overrides the +default positional arguments defined in ``tox.ini``, so you must include those +default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.6 tox_ environment using all your CPU cores:: +the Python 3.6 :doc:`tox ` environment using all your CPU cores:: tox -e py36 -- scrapy tests -n auto @@ -275,7 +276,4 @@ And their unit-tests are in:: .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/en/articles/creating-a-pull-request -.. _pytest: https://docs.pytest.org/en/latest/usage.html -.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html -.. _tox: https://pypi.python.org/pypi/tox -.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode +.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 0aaad0c77..4b2588518 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the of the spider at the method level. It has the advantage of being flexible and simple to use, but does not allow debugging code inside a method. +.. highlight:: none + +.. skip: start + In order to see the item scraped from a specific url:: $ scrapy parse --spider=myspider -c parse_item -d 2 @@ -85,6 +89,8 @@ using:: $ scrapy parse --spider=myspider -d 3 'http://example.com/page1' +.. skip: end + Scrapy Shell ============ @@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides showing the response received and the output. How to debug the situation when ``parse_details`` sometimes receives no item? +.. highlight:: python + Fortunately, the :command:`shell` is your bread and butter in this case (see :ref:`topics-shell-inspect-response`):: diff --git a/pytest.ini b/pytest.ini index 03fcdda5b..33c34b8e8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,6 @@ addopts = --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst - --ignore=docs/topics/debug.rst --ignore=docs/topics/developer-tools.rst --ignore=docs/topics/dynamic-content.rst --ignore=docs/topics/items.rst @@ -21,12 +20,17 @@ addopts = --ignore=docs/utils twisted = 1 flake8-ignore = + # Files that are only meant to provide top-level imports are expected not + # to use any of their imports: + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + # Issues pending a review: # extras extras/qps-bench-server.py E261 E501 extras/qpsclient.py E501 E261 E501 # scrapy/commands scrapy/commands/__init__.py E128 E501 - scrapy/commands/check.py F401 E501 + scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 scrapy/commands/fetch.py E401 E501 E128 E502 E731 @@ -37,7 +41,6 @@ flake8-ignore = scrapy/commands/shell.py E128 E501 E502 scrapy/commands/startproject.py E502 E127 E501 E128 scrapy/commands/version.py E501 E128 - scrapy/commands/view.py F401 # scrapy/contracts scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E502 E128 @@ -46,17 +49,16 @@ flake8-ignore = scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E306 E261 E128 W504 scrapy/core/spidermw.py E501 E731 E502 E126 E226 - scrapy/core/downloader/__init__.py F401 E501 + scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E305 E241 scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 - scrapy/core/downloader/handlers/http.py F401 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126 # scrapy/downloadermiddlewares scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 scrapy/downloadermiddlewares/decompression.py E501 @@ -66,19 +68,18 @@ flake8-ignore = scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 W504 scrapy/downloadermiddlewares/retry.py E501 E126 - scrapy/downloadermiddlewares/robotstxt.py F401 E501 + scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions scrapy/extensions/closespider.py E501 E502 E128 E123 scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 E303 F401 + scrapy/extensions/httpcache.py E128 E501 E303 scrapy/extensions/memdebug.py E501 scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 W504 scrapy/extensions/throttle.py E501 # scrapy/http - scrapy/http/__init__.py F401 scrapy/http/common.py E501 scrapy/http/cookies.py E501 scrapy/http/request/__init__.py E501 @@ -87,7 +88,7 @@ flake8-ignore = scrapy/http/response/__init__.py E501 E128 W293 W291 scrapy/http/response/text.py E501 W293 E128 E124 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401 + scrapy/linkextractors/__init__.py E731 E502 E501 E402 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 # scrapy/loader scrapy/loader/__init__.py E501 E502 E128 @@ -97,8 +98,8 @@ flake8-ignore = scrapy/pipelines/images.py E265 E501 scrapy/pipelines/media.py E125 E501 E266 # scrapy/selector - scrapy/selector/__init__.py F403 F401 - scrapy/selector/unified.py F401 E501 E111 + scrapy/selector/__init__.py F403 + scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 scrapy/settings/default_settings.py E501 E261 E114 E116 E226 @@ -106,32 +107,30 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504 + scrapy/spidermiddlewares/referer.py E501 E129 W503 W504 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders - scrapy/spiders/__init__.py F401 E501 E402 + scrapy/spiders/__init__.py E501 E402 scrapy/spiders/crawl.py E501 scrapy/spiders/feed.py E501 E261 scrapy/spiders/sitemap.py E501 # scrapy/utils scrapy/utils/benchserver.py E501 - scrapy/utils/boto.py F401 scrapy/utils/conf.py E402 E502 E501 - scrapy/utils/console.py E261 F401 E306 E305 - scrapy/utils/curl.py F401 + scrapy/utils/console.py E261 E306 E305 scrapy/utils/datatypes.py E501 E226 scrapy/utils/decorators.py E501 scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/engine.py F401 E261 + scrapy/utils/engine.py E261 scrapy/utils/gz.py E305 E501 W504 - scrapy/utils/http.py F403 F401 E226 + scrapy/utils/http.py F403 E226 scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 scrapy/utils/log.py E128 W503 - scrapy/utils/markup.py F403 F401 W292 + scrapy/utils/markup.py F403 W292 scrapy/utils/misc.py E501 E226 - scrapy/utils/multipart.py F403 F401 W292 + scrapy/utils/multipart.py F403 W292 scrapy/utils/project.py E501 scrapy/utils/python.py E501 scrapy/utils/reactor.py E226 @@ -143,7 +142,7 @@ flake8-ignore = scrapy/utils/spider.py E271 E501 scrapy/utils/ssl.py E501 scrapy/utils/test.py E501 - scrapy/utils/url.py E501 F403 F401 E128 F405 + scrapy/utils/url.py E501 F403 E128 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/_monkeypatches.py W293 @@ -167,29 +166,29 @@ flake8-ignore = scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests - tests/__init__.py F401 E402 E501 - tests/mockserver.py E401 E501 E126 E123 F401 + tests/__init__.py E402 E501 + tests/mockserver.py E401 E501 E126 E123 tests/pipelines.py F841 E226 tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 tests/test_command_fetch.py E501 E261 - tests/test_command_parse.py F401 E501 E128 E303 E226 + tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 - tests/test_commands.py F401 E128 E501 + tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 W293 tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 tests/test_dependencies.py F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123 + tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 E305 F401 - tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123 - tests/test_downloadermiddleware_httpproxy.py F401 E501 E128 + tests/test_downloadermiddleware_httpcache.py E501 E305 + tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 + tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126 tests/test_downloadermiddleware_robotstxt.py E501 @@ -197,11 +196,11 @@ flake8-ignore = tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 tests/test_engine.py E401 E501 E502 E128 E261 tests/test_exporters.py E501 E731 E306 E128 E124 - tests/test_extension_telnet.py F401 F841 - tests/test_feedexport.py E501 F401 F841 E241 + tests/test_extension_telnet.py F841 + tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 + tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 tests/test_http_response.py E501 E301 E502 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 @@ -211,21 +210,21 @@ flake8-ignore = tests/test_mail.py E128 E501 E305 tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226 - tests/test_pipeline_images.py F401 F841 E501 E303 + tests/test_pipeline_files.py E501 W293 E303 E272 E226 + tests/test_pipeline_images.py F841 E501 E303 tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 E305 - tests/test_robotstxt_interface.py F401 E501 W291 E501 + tests/test_robotstxt_interface.py E501 W291 E501 tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py F401 E501 E127 - tests/test_spider.py E501 F401 + tests/test_selector.py E501 E127 + tests/test_spider.py E501 tests/test_spidermiddleware.py E501 E226 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 - tests/test_spidermiddleware_output_chain.py F401 E501 W293 E226 - tests/test_spidermiddleware_referer.py F401 E501 F841 E125 E201 E261 E124 E501 E241 E121 + tests/test_spidermiddleware_output_chain.py E501 W293 E226 + tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121 tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 tests/test_utils_curl.py E501 @@ -236,16 +235,16 @@ flake8-ignore = tests/test_utils_iterators.py E501 E128 E129 E303 E241 tests/test_utils_log.py E741 E226 tests/test_utils_python.py E501 E303 E731 E701 E305 - tests/test_utils_reqser.py F401 E501 E128 + tests/test_utils_reqser.py E501 E128 tests/test_utils_request.py E501 E128 E305 tests/test_utils_response.py E501 tests/test_utils_signal.py E741 F841 E731 E226 tests/test_utils_sitemap.py E128 E501 E124 tests/test_utils_spider.py E261 E305 tests/test_utils_template.py E305 - tests/test_utils_url.py F401 E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 + tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 tests/test_cmdline/__init__.py E502 E501 - tests/test_settings/__init__.py F401 E501 E128 + tests/test_settings/__init__.py E501 E128 tests/test_spiderloader/__init__.py E128 E501 tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index ad241c59f..9d4437a47 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,5 +1,4 @@ import time -import sys from collections import defaultdict from unittest import TextTestRunner, TextTestResult as _TextTestResult diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 31c17c0ab..41e77ba3b 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,4 +1,4 @@ -from scrapy.commands import fetch, ScrapyCommand +from scrapy.commands import fetch from scrapy.utils.response import open_in_browser diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 73d84664f..100af9d31 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,5 +1,4 @@ import random -import warnings from time import time from datetime import datetime from collections import deque diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 85b733228..0d083caac 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -21,7 +21,7 @@ def _get_boto_connection(): return http_request.headers try: - import boto.auth + import boto.auth # noqa: F401 except ImportError: _S3Connection = _v19_S3Connection else: diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index b98ec218f..91850683f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -5,17 +5,15 @@ import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time -from warnings import warn from weakref import WeakKeyDictionary from w3lib.http import headers_raw_to_dict, headers_dict_to_raw -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path -from scrapy.utils.python import to_bytes, to_unicode, garbage_collect +from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.request import request_fingerprint diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index f2b5ea2ac..f90f50d21 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -118,4 +118,4 @@ class FilteringLinkExtractor(object): # Top-level imports -from .lxmlhtml import LxmlLinkExtractor as LinkExtractor +from .lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 90e96ee92..a9240c1f6 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,4 +1,4 @@ """ Selectors """ -from scrapy.selector.unified import * +from scrapy.selector.unified import * # noqa: F401 diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 62fda40b0..a08955dc9 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,12 +2,10 @@ XPath selectors based on lxml """ -import warnings from parsel import Selector as _ParselSelector from scrapy.utils.trackref import object_ref from scrapy.utils.python import to_bytes from scrapy.http import HtmlResponse, XmlResponse -from scrapy.utils.decorators import deprecated __all__ = ['Selector', 'SelectorList'] diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 94095bc27..8d15dfceb 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -10,7 +10,6 @@ from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import method_is_overridden @@ -100,6 +99,6 @@ class Spider(object_ref): # Top-level imports -from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.spiders.sitemap import SitemapSpider +from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401 +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401 +from scrapy.spiders.sitemap import SitemapSpider # noqa: F401 diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 46816b54d..12321caa5 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -5,7 +5,7 @@ from scrapy.exceptions import NotConfigured def is_botocore(): try: - import botocore + import botocore # noqa: F401 return True except ImportError: raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index a26e84d38..688e28c34 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -52,7 +52,7 @@ def _embed_standard_shell(namespace={}, banner=''): except ImportError: pass else: - import rlcompleter + import rlcompleter # noqa: F401 readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) def wrapper(namespace=namespace, banner=''): diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 534d961ea..267c7ecd1 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,6 +1,7 @@ """Some debugging functions for working with the Scrapy engine""" -from time import time # used in global tests code +# used in global tests code +from time import time # noqa: F401 def get_engine_status(engine): diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index ad49ef3e9..bab262393 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -8,7 +8,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.decorators import deprecated -from w3lib.http import * +from w3lib.http import * # noqa: F401 warnings.warn("Module `scrapy.utils.http` is deprecated, " diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index a18f308a3..2455fcc16 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -6,7 +6,7 @@ For new code, always import from w3lib.html instead of this module import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * +from w3lib.html import * # noqa: F401 warnings.warn("Module `scrapy.utils.markup` is deprecated. " diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py index c2d8afd07..e81f63152 100644 --- a/scrapy/utils/multipart.py +++ b/scrapy/utils/multipart.py @@ -6,7 +6,7 @@ For new code, always import from w3lib.form instead of this module import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * +from w3lib.form import * # noqa: F401 warnings.warn("Module `scrapy.utils.multipart` is deprecated. " diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 4b48868fe..c9abb12d5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -12,7 +12,7 @@ from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code from w3lib.url import * -from w3lib.url import _safe_chars, _unquotepath +from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode diff --git a/tests/mockserver.py b/tests/mockserver.py index 71fabae0c..fe28176d4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -2,7 +2,6 @@ import json import os import random import sys -import time from subprocess import Popen, PIPE from urllib.parse import urlencode diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index a78762a64..60124b93f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -542,7 +542,7 @@ class Https11InvalidDNSPattern(Https11TestCase): def setUp(self): try: - from service_identity.exceptions import CertificateError + from service_identity.exceptions import CertificateError # noqa: F401 except ImportError: raise unittest.SkipTest("cryptography lib is too old") self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' @@ -775,7 +775,7 @@ class S3TestCase(unittest.TestCase): @contextlib.contextmanager def _mocked_date(self, date): try: - import botocore.auth + import botocore.auth # noqa: F401 except ImportError: yield else: @@ -840,8 +840,10 @@ class S3TestCase(unittest.TestCase): b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): - try: import botocore - except ImportError: pass + try: + import botocore # noqa: F401 + except ImportError: + pass else: raise unittest.SkipTest( 'botocore does not support overriding date with x-amz-date') diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 09df03282..9401dd66d 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -4,8 +4,6 @@ import shutil import unittest import email.utils from contextlib import contextmanager -import pytest -import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0745c8dd3..c6a823b53 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -70,7 +70,7 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli + import brotli # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse('br') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 30920b2da..36743b1de 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,10 @@ import os -import sys from functools import partial -from twisted.trial.unittest import TestCase, SkipTest +from twisted.trial.unittest import TestCase from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request +from scrapy.http import Request from scrapy.spiders import Spider from scrapy.crawler import Crawler from scrapy.settings import Settings diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 875ceb83c..873a97248 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -3,7 +3,7 @@ from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials from twisted.internet import defer -from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 8d2b97a2f..2ca57c19d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -165,7 +165,7 @@ class S3FeedStorageTest(unittest.TestCase): create=True) def test_parse_credentials(self): try: - import boto + import boto # noqa: F401 except ImportError: raise unittest.SkipTest("S3FeedStorage requires boto") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', @@ -266,7 +266,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): try: - import botocore + import botocore # noqa: F401 except ImportError: raise unittest.SkipTest('botocore is required') @@ -286,7 +286,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): try: - import botocore + import botocore # noqa: F401 except ImportError: raise unittest.SkipTest('botocore is required') diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 01f61e7e5..a98aa1e6f 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -import cgi import unittest import re import json diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 4f7265763..7f1cb4a11 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,7 +1,6 @@ import io import hashlib import random -import warnings from tempfile import mkdtemp from shutil import rmtree diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 7a7ec4281..5f68cd448 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -2,18 +2,19 @@ import json import os import re import sys -from urllib.parse import urlsplit, urlunsplit from subprocess import Popen, PIPE +from urllib.parse import urlsplit, urlunsplit import pytest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_crawler from scrapy.http import Request -from tests.spiders import SimpleSpider, SingleRequestSpider +from scrapy.utils.test import get_crawler + from tests.mockserver import MockServer +from tests.spiders import SimpleSpider, SingleRequestSpider class MitmProxy: @@ -75,35 +76,35 @@ class ProxyConnectTestCase(TestCase): @defer.inlineCallbacks def test_https_connect_tunnel(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as logs: + with LogCapture() as l: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, logs) + self._assert_got_response_code(200, l) @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) - with LogCapture() as logs: + with LogCapture() as l: yield crawler.crawl("https://localhost:99999/status?n=200") - self._assert_got_tunnel_error(logs) + self._assert_got_tunnel_error(l) @defer.inlineCallbacks def test_https_tunnel_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) crawler = get_crawler(SimpleSpider) - with LogCapture() as logs: + with LogCapture() as l: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) # The proxy returns a 407 error code but it does not reach the client; # he just sees a TunnelError. - self._assert_got_tunnel_error(logs) + self._assert_got_tunnel_error(l) @defer.inlineCallbacks def test_https_tunnel_without_leak_proxy_authorization_header(self): request = Request(self.mockserver.url("/echo", is_secure=True)) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as logs: + with LogCapture() as l: yield crawler.crawl(seed=request) - self._assert_got_response_code(200, logs) + self._assert_got_response_code(200, l) echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) @@ -129,9 +130,9 @@ class ProxyConnectTestCase(TestCase): def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' crawler = get_crawler(SimpleSpider) - with LogCapture() as logs: + with LogCapture() as l: yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, logs) + self._assert_got_response_code(407, l) def _assert_got_response_code(self, code, log): print(log) diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 080507276..27d79437b 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -5,7 +5,7 @@ from twisted.trial import unittest def reppy_available(): # check if reppy parser is installed try: - from reppy.robots import Robots + from reppy.robots import Robots # noqa: F401 except ImportError: return False return True @@ -14,7 +14,7 @@ def reppy_available(): def rerp_available(): # check if robotexclusionrulesparser is installed try: - from robotexclusionrulesparser import RobotExclusionRulesParser + from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401 except ImportError: return False return True @@ -23,7 +23,7 @@ def rerp_available(): def protego_available(): # check if protego parser is installed try: - from protego import Protego + from protego import Protego # noqa: F401 except ImportError: return False return True diff --git a/tests/test_selector.py b/tests/test_selector.py index b2565dd78..09c2546fb 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,9 +1,9 @@ -import warnings import weakref + from twisted.trial import unittest + from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import Selector -from lxml import etree class SelectorTestCase(unittest.TestCase): diff --git a/tests/test_spider.py b/tests/test_spider.py index c0fccfdd6..aa43e3b3a 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -15,7 +15,6 @@ from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 940e31070..5b7b5e7aa 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -6,7 +6,6 @@ from twisted.internet import defer from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import MockServerSpider class LogExceptionMiddleware: diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index ceca0d278..ecec6135d 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,6 @@ from urllib.parse import urlparse from unittest import TestCase import warnings -from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index b5729b086..06d9c004c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- import unittest -import sys from scrapy.http import Request, FormRequest from scrapy.spiders import Spider diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index c358f5b82..21e9a056a 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- import unittest -from urllib.parse import urlparse from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, - parse_url, strip_url) + add_http_if_no_scheme, guess_scheme, strip_url) __doctests__ = ['scrapy.utils.url']