diff --git a/conftest.py b/conftest.py index d5d61ddd3..d54ce155c 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,3 @@ -import six import pytest @@ -8,11 +7,10 @@ collect_ignore = [ ] -if six.PY3: - for line in open('tests/py3-ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +for line in open('tests/ignores.txt'): + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) @pytest.fixture() diff --git a/docs/conf.py b/docs/conf.py index 6ec4582b1..914d1d05f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,6 +27,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'hoverxref.extension', 'notfound.extension', 'scrapydocs', 'sphinx.ext.autodoc', @@ -274,6 +275,16 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), - 'sphinx': ('https://www.sphinx-doc.org/en/stable', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master', None), + 'tox': ('https://tox.readthedocs.io/en/latest', None), + 'twisted': ('https://twistedmatrix.com/documents/current', None), } + + +# Options for sphinx-hoverxref options +# ------------------------------------ + +hoverxref_auto_ref = True diff --git a/docs/contributing.rst b/docs/contributing.rst index c4cb605ab..988015e61 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -194,8 +194,9 @@ documentation instead of duplicating the docstring in files within the Tests ===== -Tests are implemented using the `Twisted unit-testing framework`_, running -tests requires `tox`_. +Tests are implemented using the :doc:`Twisted unit-testing framework +`. Running tests requires +:doc:`tox `. .. _running-tests: @@ -210,37 +211,37 @@ To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` -To run the tests on a specific tox_ environment, use ``-e `` with an -environment name from ``tox.ini``. For example, to run the tests with Python -3.6 use:: +To run the tests on a specific :doc:`tox ` environment, use +``-e `` with an environment name from ``tox.ini``. For example, to run +the tests with Python 3.6 use:: tox -e py36 -You can also specify a comma-separated list of environmets, and use `tox’s -parallel mode`_ to run the tests on multiple environments in parallel:: +You can also specify a comma-separated list of environmets, and use :ref:`tox’s +parallel mode ` to run the tests on multiple environments in +parallel:: tox -e py37,py38 -p auto -To pass command-line options to pytest_, add them after ``--`` in your call to -tox_. Using ``--`` overrides the default positional arguments defined in -``tox.ini``, so you must include those default positional arguments -(``scrapy tests``) after ``--`` as well:: +To pass command-line options to :doc:`pytest `, add them after +``--`` in your call to :doc:`tox `. Using ``--`` overrides the +default positional arguments defined in ``tox.ini``, so you must include those +default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.6 tox_ environment using all your CPU cores:: +the Python 3.6 :doc:`tox ` environment using all your CPU cores:: tox -e py36 -- scrapy tests -n auto -To see coverage report install `coverage`_ (``pip install coverage``) and run: +To see coverage report install :doc:`coverage ` +(``pip install coverage``) and run: ``coverage report`` see output of ``coverage --help`` for more options like html or xml report. -.. _coverage: https://pypi.python.org/pypi/coverage - Writing tests ------------- @@ -261,13 +262,9 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users .. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/en/articles/creating-a-pull-request -.. _pytest: https://docs.pytest.org/en/latest/usage.html -.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html -.. _tox: https://pypi.python.org/pypi/tox -.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode +.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 8b2fef065..01986b594 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').get(), 'author': quote.xpath('span/small/text()').get(), + 'text': quote.css('span.text::text').get(), } next_page = response.css('li.next a::attr("href")').get() diff --git a/docs/requirements.txt b/docs/requirements.txt index f9db85146..0ed11c4dc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,5 @@ +-r ../requirements-py3.txt Sphinx>=2.1 +sphinx-hoverxref sphinx-notfound-page sphinx_rtd_theme diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 7c8c40b5f..1c461a511 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -273,5 +273,3 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. - -.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 2effe94dc..ae25dfa2f 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -166,11 +166,10 @@ for concurrency. For more information about asynchronous programming and Twisted see these links: -* `Introduction to Deferreds in Twisted`_ +* :doc:`twisted:core/howto/defer-intro` * `Twisted - hello, asynchronous programming`_ * `Twisted Introduction - Krondo`_ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a93bee06b..5b3cd7e75 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -1,3 +1,5 @@ +.. highlight:: none + .. _topics-commands: ================= @@ -66,7 +68,9 @@ structure by default, similar to this:: The directory where the ``scrapy.cfg`` file resides is known as the *project root directory*. That file contains the name of the python module that defines -the project settings. Here is an example:: +the project settings. Here is an example: + +.. code-block:: ini [settings] default = myproject.settings @@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file:: +under ``[settings]`` in your ``scrapy.cfg`` file: + +.. code-block:: ini [settings] default = myproject1.settings @@ -277,6 +283,8 @@ check Run contract checks. +.. skip: start + Usage examples:: $ scrapy check -l @@ -294,6 +302,8 @@ Usage examples:: [FAILED] first_spider:parse >>> Returned 92 requests, expected 0..4 +.. skip: end + .. command:: list list @@ -481,6 +491,8 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +.. skip: start + Usage example:: $ scrapy parse http://www.example.com/ -c parse_item @@ -495,6 +507,8 @@ Usage example:: # Requests ----------------------------------------------------------------- [] +.. skip: end + .. command:: settings @@ -573,7 +587,9 @@ Default: ``''`` (empty string) A module to use for looking up custom Scrapy commands. This is used to add custom commands for your Scrapy project. -Example:: +Example: + +.. code-block:: python COMMANDS_MODULE = 'mybot.commands' @@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. -The following example adds ``my_command`` command:: +The following example adds ``my_command`` command: + +.. skip: next + +.. code-block:: python from setuptools import setup, find_packages diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 0aaad0c77..4b2588518 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the of the spider at the method level. It has the advantage of being flexible and simple to use, but does not allow debugging code inside a method. +.. highlight:: none + +.. skip: start + In order to see the item scraped from a specific url:: $ scrapy parse --spider=myspider -c parse_item -d 2 @@ -85,6 +89,8 @@ using:: $ scrapy parse --spider=myspider -d 3 'http://example.com/page1' +.. skip: end + Scrapy Shell ============ @@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides showing the response received and the output. How to debug the situation when ``parse_details`` sometimes receives no item? +.. highlight:: python + Fortunately, the :command:`shell` is your bread and butter in this case (see :ref:`topics-shell-inspect-response`):: diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 12eedf2cd..72bf52227 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -9,13 +9,13 @@ Sending e-mail Although Python makes sending e-mails relatively easy via the `smtplib`_ library, Scrapy provides its own facility for sending e-mails which is very -easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid -interfering with the non-blocking IO of the crawler. It also provides a -simple API for sending attachments and it's very easy to configure, with a few -:ref:`settings `. +easy to use and it's implemented using :doc:`Twisted non-blocking IO +`, to avoid interfering with the non-blocking +IO of the crawler. It also provides a simple API for sending attachments and +it's very easy to configure, with a few :ref:`settings +`. .. _smtplib: https://docs.python.org/2/library/smtplib.html -.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html Quick example ============= @@ -39,7 +39,8 @@ MailSender class reference ========================== MailSender is the preferred class to use for sending emails from Scrapy, as it -uses `Twisted non-blocking IO`_, like the rest of the framework. +uses :doc:`Twisted non-blocking IO `, like the +rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index fae18200a..cdc4953c2 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following This method is called for every item pipeline component. :meth:`process_item` must either: return a dict with data, return an :class:`~scrapy.item.Item` - (or any descendant class) object, return a `Twisted Deferred`_ or raise + (or any descendant class) object, return a + :class:`~twisted.internet.defer.Deferred` or raise :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. @@ -67,8 +68,6 @@ Additionally, they may also implement the following methods: :type crawler: :class:`~scrapy.crawler.Crawler` object -.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html - Item pipeline example ===================== @@ -166,7 +165,8 @@ method and how to clean up the resources properly.:: Take screenshot of item ----------------------- -This example demonstrates how to return Deferred_ from :meth:`process_item` method. +This example demonstrates how to return a +:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method. It uses Splash_ to render screenshot of item url. Pipeline makes request to locally running instance of Splash_. After request is downloaded and Deferred callback fires, it saves item to a file and adds filename to an item. @@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite return item .. _Splash: https://splash.readthedocs.io/en/stable/ -.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html Duplicates filter ----------------- diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 12a5e5c60..de3f38023 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -142,20 +142,6 @@ accept one (and only one) positional argument, which will be an iterable. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. -If you want to use a plain function as a processor, make sure it receives -``self`` as the first argument:: - - def lowercase_processor(self, values): - for v in values: - yield v.lower() - - class MyItemLoader(ItemLoader): - name_in = lowercase_processor - -This is because whenever a function is assigned as a class variable, it becomes -a method and would be passed the instance as the the first argument when being -called. See `this answer on stackoverflow`_ for more details. - The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -163,7 +149,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. -.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 + Declaring Item Loaders ====================== @@ -491,6 +477,8 @@ ItemLoader objects .. attribute:: item The :class:`~scrapy.item.Item` object being parsed by this Item Loader. + This is mostly used as a property so when attempting to override this + value, you may want to check out :attr:`default_item_class` first. .. attribute:: context diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 431cc6027..206e7cfa5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline: * ``success`` is a boolean which is ``True`` if the image was downloaded successfully or ``False`` if it failed for some reason - * ``file_info_or_error`` is a dict containing the following keys (if success - is ``True``) or a `Twisted Failure`_ if there was a problem. + * ``file_info_or_error`` is a dict containing the following keys (if + success is ``True``) or a :exc:`~twisted.python.failure.Failure` if + there was a problem. * ``url`` - the url where the file was downloaded from. This is the url of the request returned from the :meth:`~get_media_requests` @@ -577,5 +578,4 @@ above:: item['image_paths'] = image_paths return item -.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html .. _MD5 hash: https://en.wikipedia.org/wiki/MD5 diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a6d4f0d6d..e3e8fdc72 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -.. seealso:: `Twisted Reactor Overview`_. +.. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: @@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ee37f648e..4cf367d96 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -121,8 +121,8 @@ Request objects :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance - as first parameter. + with 404 HTTP errors and such. It receives a + :exc:`~twisted.python.failure.Failure` as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. :type errback: callable @@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing The errback of a request is a function that will be called when an exception is raise while processing it. -It receives a `Twisted Failure`_ instance as first parameter and can be -used to track connection establishment timeouts, DNS errors etc. +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. Here's an example spider logging all errors and catching some specific errors if needed:: @@ -816,5 +816,4 @@ XmlResponse objects adds encoding auto-discovering support by looking into the XML declaration line. See :attr:`TextResponse.encoding`. -.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241 diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ff07b9d55..3f29aa323 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti Deferred signal handlers ======================== -Some signals support returning `Twisted deferreds`_ from their handlers, see -the :ref:`topics-signals-ref` below to know which ones. +Some signals support returning :class:`~twisted.internet.defer.Deferred` +objects from their handlers, see the :ref:`topics-signals-ref` below to know +which ones. -.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html .. _topics-signals-ref: @@ -155,8 +155,8 @@ item_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.spiders.Spider` object - :param failure: the exception raised as a Twisted `Failure`_ object - :type failure: `Failure`_ object + :param failure: the exception raised + :type failure: twisted.python.failure.Failure spider_closed ------------- @@ -236,8 +236,8 @@ spider_error This signal does not support returning deferreds from their handlers. - :param failure: the exception raised as a Twisted `Failure`_ object - :type failure: `Failure`_ object + :param failure: the exception raised + :type failure: twisted.python.failure.Failure :param response: the response being processed when the exception was raised :type response: :class:`~scrapy.http.Response` object @@ -333,5 +333,3 @@ response_downloaded :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object - -.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html diff --git a/pytest.ini b/pytest.ini index 529ad5d27..33c34b8e8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,8 +8,6 @@ addopts = --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst - --ignore=docs/topics/commands.rst - --ignore=docs/topics/debug.rst --ignore=docs/topics/developer-tools.rst --ignore=docs/topics/dynamic-content.rst --ignore=docs/topics/items.rst @@ -22,24 +20,27 @@ addopts = --ignore=docs/utils twisted = 1 flake8-ignore = + # Files that are only meant to provide top-level imports are expected not + # to use any of their imports: + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + # Issues pending a review: # extras extras/qps-bench-server.py E261 E501 extras/qpsclient.py E501 E261 E501 # scrapy/commands scrapy/commands/__init__.py E128 E501 - scrapy/commands/check.py F401 E501 + scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731 + scrapy/commands/fetch.py E401 E501 E128 E502 E731 scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/list.py E302 scrapy/commands/parse.py E128 E501 E731 E226 scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E302 E128 + scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 scrapy/commands/startproject.py E502 E127 E501 E128 scrapy/commands/version.py E501 E128 - scrapy/commands/view.py F401 E302 # scrapy/contracts scrapy/contracts/__init__.py E501 W504 scrapy/contracts/default.py E502 E128 @@ -48,19 +49,18 @@ flake8-ignore = scrapy/core/scheduler.py E501 scrapy/core/scraper.py E501 E306 E261 E128 W504 scrapy/core/spidermw.py E501 E731 E502 E126 E226 - scrapy/core/downloader/__init__.py F401 E501 + scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E305 E241 scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 - scrapy/core/downloader/handlers/http.py F401 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126 # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226 + scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 E126 @@ -68,43 +68,38 @@ flake8-ignore = scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 W504 scrapy/downloadermiddlewares/retry.py E501 E126 - scrapy/downloadermiddlewares/robotstxt.py F401 E501 + scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions scrapy/extensions/closespider.py E501 E502 E128 E123 - scrapy/extensions/corestats.py E302 E501 + scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 E303 F401 + scrapy/extensions/httpcache.py E128 E501 E303 scrapy/extensions/memdebug.py E501 - scrapy/extensions/spiderstate.py E302 E501 + scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 W504 scrapy/extensions/throttle.py E501 # scrapy/http - scrapy/http/__init__.py F401 scrapy/http/common.py E501 scrapy/http/cookies.py E501 scrapy/http/request/__init__.py E501 scrapy/http/request/form.py E501 E123 scrapy/http/request/json_request.py E501 scrapy/http/response/__init__.py E501 E128 W293 W291 - scrapy/http/response/html.py E302 scrapy/http/response/text.py E501 W293 E128 E124 - scrapy/http/response/xml.py E302 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401 + scrapy/linkextractors/__init__.py E731 E502 E501 E402 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 # scrapy/loader scrapy/loader/__init__.py E501 E502 E128 - scrapy/loader/common.py E302 scrapy/loader/processors.py E501 # scrapy/pipelines - scrapy/pipelines/__init__.py E302 scrapy/pipelines/files.py E116 E501 E266 scrapy/pipelines/images.py E265 E501 scrapy/pipelines/media.py E125 E501 E266 # scrapy/selector - scrapy/selector/__init__.py F403 F401 - scrapy/selector/unified.py F401 E501 E111 + scrapy/selector/__init__.py F403 + scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 scrapy/settings/default_settings.py E501 E261 E114 E116 E226 @@ -112,160 +107,144 @@ flake8-ignore = # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504 + scrapy/spidermiddlewares/referer.py E501 E129 W503 W504 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders - scrapy/spiders/__init__.py F401 E501 E402 + scrapy/spiders/__init__.py E501 E402 scrapy/spiders/crawl.py E501 scrapy/spiders/feed.py E501 E261 scrapy/spiders/sitemap.py E501 # scrapy/utils scrapy/utils/benchserver.py E501 - scrapy/utils/boto.py F401 scrapy/utils/conf.py E402 E502 E501 - scrapy/utils/console.py E302 E261 F401 E306 E305 - scrapy/utils/curl.py F401 + scrapy/utils/console.py E261 E306 E305 scrapy/utils/datatypes.py E501 E226 - scrapy/utils/decorators.py E501 E302 - scrapy/utils/defer.py E501 E302 E128 + scrapy/utils/decorators.py E501 + scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/display.py E302 - scrapy/utils/engine.py F401 E261 E302 - scrapy/utils/ftp.py E302 - scrapy/utils/gz.py E305 E501 E302 W504 - scrapy/utils/http.py F403 F401 E226 - scrapy/utils/httpobj.py E302 E501 + scrapy/utils/engine.py E261 + scrapy/utils/gz.py E305 E501 W504 + scrapy/utils/http.py F403 E226 + scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 - scrapy/utils/job.py E302 scrapy/utils/log.py E128 W503 - scrapy/utils/markup.py F403 F401 W292 + scrapy/utils/markup.py F403 W292 scrapy/utils/misc.py E501 E226 - scrapy/utils/multipart.py F403 F401 W292 + scrapy/utils/multipart.py F403 W292 scrapy/utils/project.py E501 - scrapy/utils/python.py E501 E302 - scrapy/utils/reactor.py E302 E226 + scrapy/utils/python.py E501 + scrapy/utils/reactor.py E226 scrapy/utils/reqser.py E501 - scrapy/utils/request.py E302 E127 E501 - scrapy/utils/response.py E501 E302 E128 + scrapy/utils/request.py E127 E501 + scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E271 E302 E501 + scrapy/utils/spider.py E271 E501 scrapy/utils/ssl.py E501 - scrapy/utils/template.py E302 - scrapy/utils/test.py E302 E501 - scrapy/utils/url.py E501 F403 F401 E128 F405 + scrapy/utils/test.py E501 + scrapy/utils/url.py E501 F403 E128 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/_monkeypatches.py W293 scrapy/cmdline.py E502 E501 scrapy/crawler.py E501 - scrapy/dupefilters.py E302 E501 E202 - scrapy/exceptions.py E302 E501 + scrapy/dupefilters.py E501 E202 + scrapy/exceptions.py E501 scrapy/exporters.py E501 E261 E226 - scrapy/extension.py E302 - scrapy/interfaces.py E302 E501 + scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 scrapy/logformatter.py E501 W293 scrapy/mail.py E402 E128 E501 E502 scrapy/middleware.py E502 E128 E501 scrapy/pqueues.py E501 - scrapy/resolver.py E302 scrapy/responsetypes.py E128 E501 E305 - scrapy/robotstxt.py E302 E501 + scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 scrapy/spiderloader.py E225 F841 E501 E126 scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests - tests/__init__.py F401 E402 E501 - tests/mockserver.py E401 E501 E126 E123 F401 - tests/pipelines.py E302 F841 E226 - tests/spiders.py E302 E501 E127 + tests/__init__.py E402 E501 + tests/mockserver.py E401 E501 E126 E123 + tests/pipelines.py F841 E226 + tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 tests/test_command_fetch.py E501 E261 - tests/test_command_parse.py F401 E302 E501 E128 E303 E226 + tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 - tests/test_commands.py F401 E128 E501 + tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 W293 tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 - tests/test_dependencies.py E302 F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123 + tests/test_dependencies.py F841 E501 E305 + tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 - tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501 + tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401 - tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123 - tests/test_downloadermiddleware_httpproxy.py F401 E501 E128 + tests/test_downloadermiddleware_httpcache.py E501 E305 + tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 + tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124 + tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 tests/test_engine.py E401 E501 E502 E128 E261 tests/test_exporters.py E501 E731 E306 E128 E124 - tests/test_extension_telnet.py F401 F841 - tests/test_feedexport.py E501 F401 F841 E241 + tests/test_extension_telnet.py F841 + tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 - tests/test_http_headers.py E302 E501 - tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 + tests/test_http_headers.py E501 + tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 tests/test_http_response.py E501 E301 E502 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241 - tests/test_logformatter.py E128 E501 E122 E302 - tests/test_mail.py E302 E128 E501 E305 - tests/test_middleware.py E302 E501 E128 + tests/test_loader.py E501 E731 E303 E741 E128 E117 E241 + tests/test_logformatter.py E128 E501 E122 + tests/test_mail.py E128 E501 E305 + tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226 - tests/test_pipeline_images.py F401 F841 E501 E303 + tests/test_pipeline_files.py E501 W293 E303 E272 E226 + tests/test_pipeline_images.py F841 E501 E303 tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502 + tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 E302 E305 - tests/test_robotstxt_interface.py F401 E302 E501 W291 E501 + tests/test_responsetypes.py E501 E305 + tests/test_robotstxt_interface.py E501 W291 E501 tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py F401 E501 E127 - tests/test_spider.py E501 F401 + tests/test_selector.py E501 E127 + tests/test_spider.py E501 tests/test_spidermiddleware.py E501 E226 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 - tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293 - tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226 - tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121 - tests/test_squeues.py E501 E302 E701 E741 + tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 + tests/test_spidermiddleware_output_chain.py E501 W293 E226 + tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121 + tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 - tests/test_utils_console.py E302 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E261 E501 E302 F841 E226 + tests/test_utils_defer.py E306 E261 E501 F841 E226 tests/test_utils_deprecate.py F841 E306 E501 - tests/test_utils_http.py E302 E501 E502 E128 W504 - tests/test_utils_httpobj.py E302 - tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241 + tests/test_utils_http.py E501 E502 E128 W504 + tests/test_utils_iterators.py E501 E128 E129 E303 E241 tests/test_utils_log.py E741 E226 tests/test_utils_python.py E501 E303 E731 E701 E305 - tests/test_utils_reqser.py F401 E501 E128 - tests/test_utils_request.py E302 E501 E128 E305 + tests/test_utils_reqser.py E501 E128 + tests/test_utils_request.py E501 E128 E305 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 E302 E731 E226 - tests/test_utils_sitemap.py E302 E128 E501 E124 - tests/test_utils_spider.py E261 E302 E305 + tests/test_utils_signal.py E741 F841 E731 E226 + tests/test_utils_sitemap.py E128 E501 E124 + tests/test_utils_spider.py E261 E305 tests/test_utils_template.py E305 - tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123 + tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 - tests/mocks/dummydbm.py E302 tests/test_cmdline/__init__.py E502 E501 - tests/test_cmdline/extensions.py E302 - tests/test_settings/__init__.py F401 E501 E128 - tests/test_spiderloader/__init__.py E128 E501 E302 - tests/test_spiderloader/test_spiders/spider0.py E302 - tests/test_spiderloader/test_spiders/spider1.py E302 - tests/test_spiderloader/test_spiders/spider2.py E302 - tests/test_spiderloader/test_spiders/spider3.py E302 - tests/test_spiderloader/test_spiders/nested/spider4.py E302 + tests/test_settings/__init__.py E501 E128 + tests/test_spiderloader/__init__.py E128 E501 tests/test_utils_misc/__init__.py E501 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 230e5cee3..fb8357f3c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') del warnings # Apply monkey patches to fix issues in external libraries -from . import _monkeypatches +from scrapy import _monkeypatches del _monkeypatches from twisted import version as _txv diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 3e6c11b7d..9d4437a47 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,6 +1,4 @@ -from __future__ import print_function import time -import sys from collections import defaultdict from unittest import TextTestRunner, TextTestResult as _TextTestResult diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d45133e0e..724b4a1c4 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -8,6 +8,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a255b3b94..422183ac1 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,7 @@ from __future__ import print_function from scrapy.commands import ScrapyCommand + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bee52f06a..ffe3aa2eb 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -4,6 +4,7 @@ import json from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 59e665016..41e77ba3b 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,6 +1,7 @@ -from scrapy.commands import fetch, ScrapyCommand +from scrapy.commands import fetch from scrapy.utils.response import open_in_browser + class Command(fetch.Command): def short_desc(self): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 24f6c2e77..e0d425874 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -4,7 +4,7 @@ from scrapy.item import BaseItem from scrapy.http import Request from scrapy.exceptions import ContractFail -from . import Contract +from scrapy.contracts import Contract # contracts diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 949dacbc8..c5474a57f 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,6 +1,4 @@ -from __future__ import absolute_import import random -import warnings from time import time from datetime import datetime from collections import deque @@ -12,8 +10,8 @@ from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache from scrapy import signals -from .middleware import DownloaderMiddlewareManager -from .handlers import DownloadHandlers +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.core.downloader.handlers import DownloadHandlers class Slot(object): diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 89d2776ae..6e023ebcc 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): """ Twisted-recommended context factory for web clients. - Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: - "The default is to use a BrowserLikePolicyForHTTPS, - so unless you have special requirements you can leave this as-is." + Quoting the documentation of the :class:`~twisted.web.client.Agent` class: - creatorForNetloc() is the same as BrowserLikePolicyForHTTPS - except this context factory allows setting the TLS/SSL method to use. + The default is to use a + :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you + have special requirements you can leave this as-is. - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation. + :meth:`creatorForNetloc` is the same as + :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context + factory allows setting the TLS/SSL method to use. + + The default OpenSSL method is ``TLS_METHOD`` (also called + ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ def creatorForNetloc(self, hostname, port): diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index ad25beb3b..9e5020753 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -17,8 +17,8 @@ class DataURIDownloadHandler(object): respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} - if (issubclass(respcls, TextResponse) and - uri.media_type.split('/')[0] == 'text'): + if (issubclass(respcls, TextResponse) + and uri.media_type.split('/')[0] == 'text'): charset = uri.media_type_parameters.get('charset') resp_kwargs['encoding'] = charset diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index ac4b867c3..52535bd8b 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,3 +1,4 @@ -from __future__ import absolute_import -from .http10 import HTTP10DownloadHandler -from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.core.downloader.handlers.http11 import ( + HTTP11DownloadHandler as HTTPDownloadHandler, +) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d8bbdd326..f4a42ce12 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -3,7 +3,7 @@ from six.moves.urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.boto import is_botocore -from .http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler def _get_boto_connection(): @@ -21,7 +21,7 @@ def _get_boto_connection(): return http_request.headers try: - import boto.auth + import boto.auth # noqa: F401 except ImportError: _S3Connection = _v19_S3Connection else: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8868a985b..19b61dc7e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,3 +1,4 @@ +import pprint import six import signal import logging @@ -45,7 +46,8 @@ class Crawler(object): logging.root.addHandler(handler) d = dict(overridden_settings(self.settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) + logger.info("Overridden settings:\n%(settings)s", + {'settings': pprint.pformat(d)}) if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings @@ -110,7 +112,7 @@ class Crawler(object): class CrawlerRunner(object): """ This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup Twisted `reactor`_. + crawlers inside an already setup :mod:`~twisted.internet.reactor`. The CrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. @@ -233,12 +235,13 @@ class CrawlerProcess(CrawlerRunner): A class to run multiple scrapy crawlers in a process simultaneously. This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a Twisted `reactor`_ and handling shutdown signals, like the - keyboard interrupt command Ctrl-C. It also configures top-level logging. + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. This utility should be a better fit than :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - Twisted `reactor`_ within your application. + :mod:`~twisted.internet.reactor` within your application. The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. @@ -273,9 +276,9 @@ class CrawlerProcess(CrawlerRunner): def start(self, stop_after_crawl=True): """ - This method starts a Twisted `reactor`_, adjusts its pool size to - :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based - on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache + based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 72715dba7..ba50793bb 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -68,6 +68,8 @@ class AjaxCrawlMiddleware(object): # XXX: move it to w3lib? _ajax_crawlable_re = re.compile(six.u(r'')) + + def _has_ajaxcrawlable_meta(text): """ >>> _has_ajaxcrawlable_meta('') diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 0d2b9900c..aeb7578b8 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,8 +1,8 @@ -import os -import six import logging from collections import defaultdict +import six + from scrapy.exceptions import NotConfigured from scrapy.http import Response from scrapy.http.cookies import CookieJar diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 495b103d1..4e06f8236 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,11 +1,19 @@ from email.utils import formatdate + from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed + from scrapy import signals -from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2212d9688..5e4542b6c 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,8 @@ import base64 +from urllib.request import _parse_proxy + from six.moves.urllib.parse import unquote, urlunparse from six.moves.urllib.request import getproxies, proxy_bypass -from urllib.request import _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0bcdd3495..4d95eb847 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -5,6 +5,7 @@ import logging from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint + class BaseDupeFilter(object): @classmethod diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 96949bdd9..7c4bb3d00 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -7,10 +7,12 @@ new exceptions here without documenting them there. # Internal + class NotConfigured(Exception): """Indicates a missing configuration situation""" pass + class _InvalidOutput(TypeError): """ Indicates an invalid value has been returned by a middleware's processing method. @@ -18,15 +20,19 @@ class _InvalidOutput(TypeError): """ pass + # HTTP and crawling + class IgnoreRequest(Exception): """Indicates a decision was made not to process a request""" + class DontCloseSpider(Exception): """Request the spider not to be closed yet""" pass + class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -34,30 +40,37 @@ class CloseSpider(Exception): super(CloseSpider, self).__init__() self.reason = reason + # Items + class DropItem(Exception): """Drop item from the item pipeline""" pass + class NotSupported(Exception): """Indicates a feature or method is not supported""" pass + # Commands + class UsageError(Exception): """To indicate a command-line usage error""" def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) super(UsageError, self).__init__(*a, **kw) + class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default DeprecationWarning is silenced on Python 2.7+ """ pass + class ContractFail(AssertionError): """Error raised in case of a failing contract""" pass diff --git a/scrapy/extension.py b/scrapy/extension.py index e39e456fa..050b87e5f 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list + class ExtensionManager(MiddlewareManager): component_name = 'extension' diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index f3fabf710..11403957c 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -6,18 +6,16 @@ import os from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time -from warnings import warn from weakref import WeakKeyDictionary from six.moves import cPickle as pickle from w3lib.http import headers_raw_to_dict, headers_dict_to_raw -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path -from scrapy.utils.python import to_bytes, to_unicode, garbage_collect +from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.request import request_fingerprint diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2220cbd8f..8ba770ec0 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -5,6 +5,7 @@ from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir + class SpiderState(object): """Store and load spider state during a scraping job""" diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index bd3559fbb..7eed052c2 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class HtmlResponse(TextResponse): pass diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 1df33fee5..abf474a2f 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class XmlResponse(TextResponse): pass diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index d48babc3c..1896ec31e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,5 +1,6 @@ from zope.interface import Interface + class ISpiderLoader(Interface): def from_settings(settings): diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index e4c62f87b..8411c4d59 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -118,4 +118,4 @@ class FilteringLinkExtractor(object): # Top-level imports -from .lxmlhtml import LxmlLinkExtractor as LinkExtractor +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index e689b4727..49aa2be46 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -4,7 +4,7 @@ from six.moves.urllib.parse import urljoin from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link -from .sgml import SgmlLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 60fd6d222..7cf67e29e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,8 +4,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ from collections import defaultdict - -import six +from contextlib import suppress from scrapy.item import Item from scrapy.loader.common import wrap_loader_context @@ -15,6 +14,17 @@ from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten +def unbound_method(method): + """ + Allow to use single-argument functions as input or output processors + (no need to define an unused first 'self' argument) + """ + with suppress(AttributeError): + if '.' not in method.__qualname__: + return method.__func__ + return method + + class ItemLoader(object): default_item_class = Item @@ -72,7 +82,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k, v in six.iteritems(value): + for k, v in value.items(): self._add_value(k, v) else: self._add_value(field_name, value) @@ -82,7 +92,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k, v in six.iteritems(value): + for k, v in value.items(): self._replace_value(k, v) else: self._replace_value(field_name, value) @@ -142,14 +152,14 @@ class ItemLoader(object): if not proc: proc = self._get_item_field_attr(field_name, 'input_processor', self.default_input_processor) - return proc + return unbound_method(proc) def get_output_processor(self, field_name): proc = getattr(self, '%s_out' % field_name, None) if not proc: proc = self._get_item_field_attr(field_name, 'output_processor', self.default_output_processor) - return proc + return unbound_method(proc) def _process_input_value(self, field_name, value): proc = self.get_input_processor(field_name) diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py index 916524947..42f8de636 100644 --- a/scrapy/loader/common.py +++ b/scrapy/loader/common.py @@ -3,6 +3,7 @@ from functools import partial from scrapy.utils.python import get_func_args + def wrap_loader_context(function, context): """Wrap functions that receive loader_context to contain the context "pre-loaded" and expose a interface that receives only one argument diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 2ef8786d0..aa1bfb77f 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rst from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list + class ItemPipelineManager(MiddlewareManager): component_name = 'item pipeline' diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0aaced7e4..4df949015 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCache dnscache = LocalCache(10000) + class CachingThreadedResolver(ThreadedResolver): def __init__(self, reactor, cache_size, timeout): super(CachingThreadedResolver, self).__init__(reactor) diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 95a8c09b8..f0f9c59dc 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -5,8 +5,10 @@ from six import with_metaclass from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) + def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: @@ -23,6 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): robotstxt_body = '' return robotstxt_body + class RobotParser(with_metaclass(ABCMeta)): @classmethod @abstractmethod diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 90e96ee92..a9240c1f6 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,4 +1,4 @@ """ Selectors """ -from scrapy.selector.unified import * +from scrapy.selector.unified import * # noqa: F401 diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 62fda40b0..a08955dc9 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,12 +2,10 @@ XPath selectors based on lxml """ -import warnings from parsel import Selector as _ParselSelector from scrapy.utils.trackref import object_ref from scrapy.utils.python import to_bytes from scrapy.http import HtmlResponse, XmlResponse -from scrapy.utils.decorators import deprecated __all__ = ['Selector', 'SelectorList'] diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..9a160f62e 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -46,16 +46,14 @@ class SignalManager(object): def send_catch_log_deferred(self, signal, **kwargs): """ - Like :meth:`send_catch_log` but supports returning `deferreds`_ from - signal handlers. + Like :meth:`send_catch_log` but supports returning + :class:`~twisted.internet.defer.Deferred` objects from signal handlers. Returns a Deferred that gets fired once all signal handlers deferreds were fired. Send a signal, catch exceptions and log them. The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). - - .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html """ kwargs.setdefault('sender', self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 94095bc27..9429f6cb2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -10,7 +10,6 @@ from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import method_is_overridden @@ -58,6 +57,11 @@ class Spider(object_ref): def start_requests(self): cls = self.__class__ + if not self.start_urls and hasattr(self, 'start_url'): + raise AttributeError( + "Crawling could not start: 'start_urls' not found " + "or empty (but found 'start_url' attribute instead, " + "did you miss an 's'?)") if method_is_overridden(cls, Spider, 'make_requests_from_url'): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " @@ -100,6 +104,6 @@ class Spider(object_ref): # Top-level imports -from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.spiders.sitemap import SitemapSpider +from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401 +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401 +from scrapy.spiders.sitemap import SitemapSpider # noqa: F401 diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index c8fc911bb..b76d5e56e 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -7,7 +7,7 @@ from scrapy.exceptions import NotConfigured def is_botocore(): try: - import botocore + import botocore # noqa: F401 return True except ImportError: raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 2e9981556..688e28c34 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,6 +1,7 @@ from functools import wraps from collections import OrderedDict + def _embed_ipython_shell(namespace={}, banner=''): """Start an IPython Shell""" try: @@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''): shell() return wrapper + def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython @@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''): bpython.embed(locals_=namespace, banner=banner) return wrapper + def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl @@ -40,6 +43,7 @@ def _embed_ptpython_shell(namespace={}, banner=''): ptpython.repl.embed(locals=namespace) return wrapper + def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code @@ -48,13 +52,14 @@ def _embed_standard_shell(namespace={}, banner=''): except ImportError: pass else: - import rlcompleter + import rlcompleter # noqa: F401 readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) def wrapper(namespace=namespace, banner=''): code.interact(banner=banner, local=namespace) return wrapper + DEFAULT_PYTHON_SHELLS = OrderedDict([ ('ptpython', _embed_ptpython_shell), ('ipython', _embed_ipython_shell), @@ -62,6 +67,7 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([ ('python', _embed_standard_shell), ]) + def get_shell_embed_func(shells=None, known_shells=None): """Return the first acceptable shell-embed function from a given list of shell names. @@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None): except ImportError: continue + def start_python_console(namespace=None, banner='', shells=None): """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index b3fd0a497..7fb25a71d 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -4,7 +4,7 @@ from shlex import split from six.moves.http_cookies import SimpleCookie from six.moves.urllib.parse import urlparse -from six import string_types, iteritems +from six import iteritems from w3lib.http import basic_auth_header diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 38bee1a6c..2e2c7adc1 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -34,6 +34,7 @@ def defers(func): return defer.maybeDeferred(func, *a, **kw) return wrapped + def inthread(func): """Decorator to call a function in a thread and return a deferred with the result diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 69d621830..c5916c21c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -7,6 +7,7 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest + def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -18,6 +19,7 @@ def defer_fail(_failure): reactor.callLater(0.1, d.errback, _failure) return d + def defer_succeed(result): """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -29,6 +31,7 @@ def defer_succeed(result): reactor.callLater(0.1, d.callback, result) return d + def defer_result(result): if isinstance(result, defer.Deferred): return result @@ -37,6 +40,7 @@ def defer_result(result): else: return defer_succeed(result) + def mustbe_deferred(f, *args, **kw): """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop @@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw): else: return defer_result(result) + def parallel(iterable, count, callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named): work = (callable(elem, *args, **named) for elem in iterable) return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + def process_chain(callbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() @@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw): d.callback(input) return d + def process_chain_both(callbacks, errbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() @@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): d.callback(input) return d + def process_parallel(callbacks, input, *a, **kw): """Return a Deferred with the output of all successful calls to the given callbacks @@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw): d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d + def iter_errback(iterable, errback, *a, **kw): """Wraps an iterable calling an errback if an error is caught while iterating it. diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f6a6c4645..91ebdae11 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -6,6 +6,7 @@ from __future__ import print_function import sys from pprint import pformat as pformat_ + def _colorize(text, colorize=True): if not colorize or not sys.stdout.isatty(): return text @@ -17,8 +18,10 @@ def _colorize(text, colorize=True): except ImportError: return text + def pformat(obj, *args, **kwargs): return _colorize(pformat_(obj), kwargs.pop('colorize', True)) + def pprint(obj, *args, **kwargs): print(pformat(obj, *args, **kwargs)) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 11dd36d91..267c7ecd1 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,7 +1,8 @@ """Some debugging functions for working with the Scrapy engine""" -from __future__ import print_function -from time import time # used in global tests code +# used in global tests code +from time import time # noqa: F401 + def get_engine_status(engine): """Return a report of the current engine status""" @@ -32,6 +33,7 @@ def get_engine_status(engine): return checks + def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" @@ -41,5 +43,6 @@ def format_engine_status(engine=None): return s + def print_engine_status(engine): print(format_engine_status(engine)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 9eca6a4da..91d2439a9 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,6 +1,7 @@ from ftplib import error_perm from posixpath import dirname + def ftp_makedirs_cwd(ftp, path, first_call=True): """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index f41e62fe3..9672e28da 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -45,6 +45,7 @@ def gunzip(data): _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search + @deprecated def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index ad49ef3e9..bab262393 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -8,7 +8,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.decorators import deprecated -from w3lib.http import * +from w3lib.http import * # noqa: F401 warnings.warn("Module `scrapy.utils.http` is deprecated, " diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index b4c929b0e..b2be0a901 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -4,7 +4,10 @@ import weakref from six.moves.urllib.parse import urlparse + _urlparse_cache = weakref.WeakKeyDictionary() + + def urlparse_cached(request_or_response): """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 389fde73a..4f1e601fc 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,5 +1,6 @@ import os + def job_dir(settings): path = settings['JOBDIR'] if path and not os.path.exists(path): diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index a18f308a3..2455fcc16 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -6,7 +6,7 @@ For new code, always import from w3lib.html instead of this module import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * +from w3lib.html import * # noqa: F401 warnings.warn("Module `scrapy.utils.markup` is deprecated. " diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py index c2d8afd07..e81f63152 100644 --- a/scrapy/utils/multipart.py +++ b/scrapy/utils/multipart.py @@ -6,7 +6,7 @@ For new code, always import from w3lib.form instead of this module import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * +from w3lib.form import * # noqa: F401 warnings.warn("Module `scrapy.utils.multipart` is deprecated. " diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 663a8ebaa..138e86d37 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -165,6 +165,7 @@ def memoizemethod_noargs(method): _BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} + @deprecated("scrapy.utils.python.binary_is_text") def isbinarytext(text): """ This function is deprecated. @@ -382,9 +383,11 @@ class MutableChain(object): self.data = chain(self.data, *iterables) def __iter__(self): - return self.data.__iter__() + return self def __next__(self): return next(self.data) - next = __next__ + @deprecated("scrapy.utils.python.MutableChain.__next__") + def next(self): + return self.__next__() diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index eda7867e3..493d26d4c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,5 +1,6 @@ from twisted.internet import reactor, error + def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" assert len(portrange) <= 2, "invalid portrange: %s" % portrange diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 63d0ae772..0fce5a2e1 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -16,6 +16,8 @@ from scrapy.utils.httpobj import urlparse_cached _fingerprint_cache = weakref.WeakKeyDictionary() + + def request_fingerprint(request, include_headers=None, keep_fragments=False): """ Return the request fingerprint. diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index feab07431..29fdaaf2c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -13,6 +13,8 @@ from w3lib import html _baseurl_cache = weakref.WeakKeyDictionary() + + def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: @@ -23,6 +25,8 @@ def get_base_url(response): _metaref_cache = weakref.WeakKeyDictionary() + + def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 94b24f67e..bf4973fbf 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -28,6 +28,7 @@ def iter_spider_classes(module): getattr(obj, 'name', None): yield obj + def spidercls_for_request(spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False): """Return a spider class that handles the given Request. diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 615372fc8..96ff4b09b 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') + + def string_camelcase(string): """ Convert a word to its CamelCase version and remove invalid chars diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..9754366df 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -32,6 +32,7 @@ def skip_if_no_boto(): except NotConfigured as e: raise SkipTest(e) + def get_s3_content_and_delete(bucket, path, with_key=False): """ Get content from s3 key, and delete key afterwards. """ @@ -51,6 +52,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content + def get_gcs_content_and_delete(bucket, path): from google.cloud import storage client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) @@ -61,6 +63,7 @@ def get_gcs_content_and_delete(bucket, path): bucket.delete_blob(path) return content, acl, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -72,12 +75,14 @@ def get_crawler(spidercls=None, settings_dict=None): runner = CrawlerRunner(settings_dict) return runner.create_crawler(spidercls or Spider) + def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" scrapy_path = import_module('scrapy').__path__[0] return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') + def get_testenv(): """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. @@ -86,6 +91,7 @@ def get_testenv(): env['PYTHONPATH'] = get_pythonpath() return env + def assert_samelines(testcase, text1, text2, msg=None): """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b3a4be007..2c7b324a1 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -12,7 +12,7 @@ from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code from w3lib.url import * -from w3lib.url import _safe_chars, _unquotepath +from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 48484b303..b0737d3d5 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -27,5 +27,5 @@ def scrapy_components_versions(): ("Python", sys.version.replace("\n", "- ")), ("pyOpenSSL", get_openssl_version()), ("cryptography", cryptography.__version__), - ("Platform", platform.platform()), + ("Platform", platform.platform()), ] diff --git a/tests/py3-ignores.txt b/tests/ignores.txt similarity index 74% rename from tests/py3-ignores.txt rename to tests/ignores.txt index 313e74ec9..45cf6fb92 100644 --- a/tests/py3-ignores.txt +++ b/tests/ignores.txt @@ -1,6 +1,3 @@ -tests/test_linkextractors_deprecated.py -tests/test_proxy_connect.py - scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 431428331..75c74daf5 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -13,6 +13,7 @@ error = KeyError _DATABASES = collections.defaultdict(DummyDB) + def open(file, flag='r', mode=0o666): """Open or create a dummy database compatible. diff --git a/tests/mockserver.py b/tests/mockserver.py index b766bb653..7ebb8bb62 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,9 +1,11 @@ -from __future__ import print_function -import sys, time, random, os, json -from six.moves.urllib.parse import urlencode +import json +import os +import random +import sys from subprocess import Popen, PIPE from OpenSSL import SSL +from six.moves.urllib.parse import urlencode from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.web.static import File diff --git a/tests/pipelines.py b/tests/pipelines.py index 7e2895a5c..d7d3b5259 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -2,6 +2,7 @@ Some pipelines used for testing """ + class ZeroDivisionErrorPipeline(object): def open_spider(self, spider): diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2e8d319d2..c4bc1f278 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,5 +1,7 @@ # Tests requirements jmespath +mitmproxy; python_version >= '3.6' +mitmproxy==3.0.4; python_version < '3.6' pytest pytest-cov pytest-twisted diff --git a/tests/spiders.py b/tests/spiders.py index 7816bf7c7..2487ecc22 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -16,6 +16,7 @@ class MockServerSpider(Spider): super(MockServerSpider, self).__init__(*args, **kwargs) self.mockserver = mockserver + class MetaSpider(MockServerSpider): name = 'meta' diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 28456b55d..c64e87d81 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,5 +1,6 @@ """A test extension used to check the settings loading order""" + class TestExtension(object): def __init__(self, settings): diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b134beb88..b7035fdff 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -12,6 +12,7 @@ def _textmode(bstr): and reading from it in text mode""" return to_unicode(bstr).replace(os.linesep, '\n') + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 03bf2ffcf..e31ccd9b5 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,6 +1,7 @@ from importlib import import_module from twisted.trial import unittest + class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 59d4a3eec..60124b93f 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,4 @@ import os -import six import shutil import tempfile from unittest import mock @@ -543,7 +542,7 @@ class Https11InvalidDNSPattern(Https11TestCase): def setUp(self): try: - from service_identity.exceptions import CertificateError + from service_identity.exceptions import CertificateError # noqa: F401 except ImportError: raise unittest.SkipTest("cryptography lib is too old") self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' @@ -630,18 +629,16 @@ class Http11MockServerTestCase(unittest.TestCase): # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - if six.PY2: - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url=self.mockserver.url('/xpayload')) - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure is None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') - else: - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload fails on PY3") + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url=self.mockserver.url('/xpayload')) + yield crawler.crawl(seed=request) + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure is None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') class UriResource(resource.Resource): @@ -778,7 +775,7 @@ class S3TestCase(unittest.TestCase): @contextlib.contextmanager def _mocked_date(self, date): try: - import botocore.auth + import botocore.auth # noqa: F401 except ImportError: yield else: @@ -843,8 +840,10 @@ class S3TestCase(unittest.TestCase): b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): - try: import botocore - except ImportError: pass + try: + import botocore # noqa: F401 + except ImportError: + pass else: raise unittest.SkipTest( 'botocore does not support overriding date with x-amz-date') diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 493691ea4..5a56c9db2 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -5,8 +5,10 @@ from scrapy.spiders import Spider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler + __doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl'] + class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9d863b6e3..9401dd66d 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,12 +1,9 @@ -from __future__ import print_function import time import tempfile import shutil import unittest import email.utils from contextlib import contextmanager -import pytest -import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -149,6 +146,7 @@ class FilesystemStorageTest(DefaultStorageTest): storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0745c8dd3..c6a823b53 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -70,7 +70,7 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli + import brotli # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse('br') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 30920b2da..36743b1de 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,10 @@ import os -import sys from functools import partial -from twisted.trial.unittest import TestCase, SkipTest +from twisted.trial.unittest import TestCase from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request +from scrapy.http import Request from scrapy.spiders import Spider from scrapy.crawler import Crawler from scrapy.settings import Settings diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index d7eb98c97..e4b0bdf83 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -12,6 +12,7 @@ from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider + class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 875ceb83c..873a97248 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -3,7 +3,7 @@ from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials from twisted.internet import defer -from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 87139e81f..ce3c4f059 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -167,7 +167,7 @@ class S3FeedStorageTest(unittest.TestCase): create=True) def test_parse_credentials(self): try: - import boto + import boto # noqa: F401 except ImportError: raise unittest.SkipTest("S3FeedStorage requires boto") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', @@ -268,7 +268,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): try: - import botocore + import botocore # noqa: F401 except ImportError: raise unittest.SkipTest('botocore is required') @@ -288,7 +288,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): try: - import botocore + import botocore # noqa: F401 except ImportError: raise unittest.SkipTest('botocore is required') diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 69d906fbf..c83cf3b66 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -3,6 +3,7 @@ import copy from scrapy.http import Headers + class HeadersTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5134a03b9..9df6ff67b 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -import cgi import unittest import re import json @@ -8,7 +6,7 @@ from urllib.parse import unquote_to_bytes import warnings from six.moves import xmlrpc_client as xmlrpclib -from six.moves.urllib.parse import urlparse, parse_qs, unquote +from six.moves.urllib.parse import urlparse, parse_qs from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_unicode diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py deleted file mode 100644 index 1366971be..000000000 --- a/tests/test_linkextractors_deprecated.py +++ /dev/null @@ -1,233 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -from scrapy.linkextractors.regex import RegexLinkExtractor -from scrapy.http import HtmlResponse -from scrapy.link import Link -from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor -from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor -from tests import get_testdata - -from tests.test_linkextractors import Base - - -class BaseSgmlLinkExtractorTestCase(unittest.TestCase): - # XXX: should we move some of these tests to base link extractor tests? - - def test_basic(self): - html = """Page title<title> - <body><p><a href="item/12.html">Item 12</a></p> - <p><a href="/about.html">About us</a></p> - <img src="/logo.png" alt="Company logo (not a link)" /> - <p><a href="../othercat.html">Other category</a></p> - <p><a href="/">>></a></p> - <p><a href="/" /></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), - Link(url='http://example.org/about.html', text='About us'), - Link(url='http://example.org/othercat.html', text='Other category'), - Link(url='http://example.org/', text='>>'), - Link(url='http://example.org/', text='')]) - - def test_base_url(self): - html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> - <body><p><a href="item/12.html">Item 12</a></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) - - # base url is an absolute path and relative to host - html = """<html><head><title>Page title<title><base href="/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://example.org/item/12.html', text='Item 12')]) - - # base url has no scheme - html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) - - def test_link_text_wrong_encoding(self): - html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>""" - response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'), - ]) - - def test_extraction_encoding(self): - body = get_testdata('link_extractor', 'linkextractor_noenc.html') - response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) - response_noenc = HtmlResponse(url='http://example.com/noenc', body=body) - body = get_testdata('link_extractor', 'linkextractor_latin1.html') - response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) - - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response_utf8), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) - - self.assertEqual(lx.extract_links(response_noenc), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) - - # document encoding does not affect URL path component, only query part - # >>> u'sample_ñ.html'.encode('utf8') - # b'sample_\xc3\xb1.html' - # >>> u"sample_á.html".encode('utf8') - # b'sample_\xc3\xa1.html' - # >>> u"sample_ö.html".encode('utf8') - # b'sample_\xc3\xb6.html' - # >>> u"£32".encode('latin1') - # b'\xa332' - # >>> u"µ".encode('latin1') - # b'\xb5' - self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), - Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''), - ]) - - def test_matches(self): - url1 = 'http://lotsofstuff.com/stuff1/index' - url2 = 'http://evenmorestuff.com/uglystuff/index' - - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), True) - - -class HtmlParserLinkExtractorTestCase(unittest.TestCase): - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = HtmlParserLinkExtractor() - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), - ]) - - def test_link_wrong_href(self): - html = """ - <a href="http://example.org/item1.html">Item 1</a> - <a href="http://[example.org/item2.html">Item 2</a> - <a href="http://example.org/item3.html">Item 3</a> - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = HtmlParserLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - -class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): - extractor_cls = SgmlLinkExtractor - escapes_whitespace = True - - def test_deny_extensions(self): - html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" - response = HtmlResponse("http://example.org/", body=html) - lx = SgmlLinkExtractor(deny_extensions="jpg") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), - ]) - - def test_attrs_sgml(self): - html = """<html><area href="sample1.html"></area> - <a ref="sample2.html">sample text 2</a></html>""" - response = HtmlResponse("http://example.com/index.html", body=html) - lx = SgmlLinkExtractor(attrs="href") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - ]) - - def test_link_nofollow(self): - html = """ - <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a> - <a href="about.html">About us</a> - <a href="http://google.com/something" rel="external nofollow">Something</a> - """ - response = HtmlResponse("http://example.org/page.html", body=html) - lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), - Link(url='http://example.org/about.html', text=u'About us', nofollow=False), - Link(url='http://google.com/something', text=u'Something', nofollow=True), - ]) - - -class RegexLinkExtractorTestCase(unittest.TestCase): - # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten - # not to depend on SgmlLinkExractor. Its speed is also much worse - # than it should be. - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = RegexLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) - - def test_link_wrong_href(self): - html = """ - <a href="http://example.org/item1.html">Item 1</a> - <a href="http://[example.org/item2.html">Item 2</a> - <a href="http://example.org/item3.html">Item 3</a> - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - def test_html_base_href(self): - html = """ - <html> - <head> - <base href="http://b.com/"> - </head> - <body> - <a href="test.html"></a> - </body> - </html> - """ - response = HtmlResponse("http://a.com/", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://b.com/test.html', text=u'', nofollow=False), - ]) - - @unittest.expectedFailure - def test_extraction(self): - # RegexLinkExtractor doesn't parse URLs with leading/trailing - # whitespaces correctly. - super(RegexLinkExtractorTestCase, self).test_extraction() diff --git a/tests/test_loader.py b/tests/test_loader.py index b87602809..6bfc31dbf 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -994,5 +994,53 @@ class SelectJmesTestCase(unittest.TestCase): ) +# Functions as processors + +def function_processor_strip(iterable): + return [x.strip() for x in iterable] + + +def function_processor_upper(iterable): + return [x.upper() for x in iterable] + + +class FunctionProcessorItem(Item): + foo = Field( + input_processor=function_processor_strip, + output_processor=function_processor_upper, + ) + + +class FunctionProcessorItemLoader(ItemLoader): + default_item_class = FunctionProcessorItem + + +class FunctionProcessorDictLoader(ItemLoader): + default_item_class = dict + foo_in = function_processor_strip + foo_out = function_processor_upper + + +class FunctionProcessorTestCase(unittest.TestCase): + + def test_processor_defined_in_item(self): + lo = FunctionProcessorItemLoader() + lo.add_value('foo', ' bar ') + lo.add_value('foo', [' asdf ', ' qwerty ']) + self.assertEqual( + dict(lo.load_item()), + {'foo': ['BAR', 'ASDF', 'QWERTY']} + ) + + def test_processor_defined_in_item_loader(self): + lo = FunctionProcessorDictLoader() + lo.add_value('foo', ' bar ') + lo.add_value('foo', [' asdf ', ' qwerty ']) + self.assertEqual( + dict(lo.load_item()), + {'foo': ['BAR', 'ASDF', 'QWERTY']} + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index afbd25d0c..d0b23a8c4 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -149,6 +149,7 @@ class DropSomeItemsPipeline(object): else: self.drop = True + class ShowOrSkipMessagesTestCase(TwistedTestCase): def setUp(self): self.mockserver = MockServer() diff --git a/tests/test_mail.py b/tests/test_mail.py index b139e98d8..ddb0f1e70 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -6,6 +6,7 @@ from email.charset import Charset from scrapy.mail import MailSender + class MailSenderTest(unittest.TestCase): def test_send(self): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index af9b43d61..ebf817c7e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -4,6 +4,7 @@ from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager + class M1(object): def open_spider(self, spider): @@ -15,6 +16,7 @@ class M1(object): def process(self, response, request, spider): pass + class M2(object): def open_spider(self, spider): @@ -25,6 +27,7 @@ class M2(object): pass + class M3(object): def process(self, response, request, spider): @@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process'): self.methods['process'].append(mw.process) + class MiddlewareManagerTest(unittest.TestCase): def test_init(self): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 4f7265763..7f1cb4a11 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,7 +1,6 @@ import io import hashlib import random -import warnings from tempfile import mkdtemp from shutil import rmtree diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index ae1236bcb..277455751 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,38 +1,53 @@ import json import os -import time +import re +from subprocess import Popen, PIPE +import sys +import pytest from six.moves.urllib.parse import urlsplit, urlunsplit -from threading import Thread -from libmproxy import controller, proxy -from netlib import http_auth from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase + from scrapy.utils.test import get_crawler from scrapy.http import Request from tests.spiders import SimpleSpider, SingleRequestSpider from tests.mockserver import MockServer -class HTTPSProxy(controller.Master, Thread): +class MitmProxy: + auth_user = 'scrapy' + auth_pass = 'scrapy' - def __init__(self): - password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') - authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + def start(self): + from scrapy.utils.test import get_testenv + script = """ +import sys +from mitmproxy.tools.main import mitmdump +sys.argv[0] = "mitmdump" +sys.exit(mitmdump()) + """ cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') - server = proxy.ProxyServer(proxy.ProxyConfig( - authenticator = authenticator, - cacert = cert_path), - 0) - self.server = server - Thread.__init__(self) - controller.Master.__init__(self, server) + 'keys', 'mitmproxy-ca.pem') + self.proc = Popen([sys.executable, + '-c', script, + '--listen-host', '127.0.0.1', + '--listen-port', '0', + '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--certs', cert_path, + '--ssl-insecure', + ], + stdout=PIPE, env=get_testenv()) + line = self.proc.stdout.readline().decode('utf-8') + host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) + address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + return address - def http_address(self): - return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + def stop(self): + self.proc.kill() + self.proc.communicate() def _wrong_credentials(proxy_url): @@ -40,6 +55,7 @@ def _wrong_credentials(proxy_url): bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') return urlunsplit(bad_auth_proxy) + class ProxyConnectTestCase(TestCase): def setUp(self): @@ -47,17 +63,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy() - self._proxy.start() - - # Wait for the proxy to start. - time.sleep(1.0) - os.environ['https_proxy'] = self._proxy.http_address() - os.environ['http_proxy'] = self._proxy.http_address() + self._proxy = MitmProxy() + proxy_url = self._proxy.start() + os.environ['https_proxy'] = proxy_url + os.environ['http_proxy'] = proxy_url def tearDown(self): self.mockserver.__exit__(None, None, None) - self._proxy.shutdown() + self._proxy.stop() os.environ = self._oldenv @defer.inlineCallbacks @@ -67,15 +80,7 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - @defer.inlineCallbacks - def test_https_noconnect(self): - proxy = os.environ['https_proxy'] - os.environ['https_proxy'] = proxy + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - + @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) @@ -100,9 +105,27 @@ class ProxyConnectTestCase(TestCase): with LogCapture() as l: yield crawler.crawl(seed=request) self._assert_got_response_code(200, l) - echo = json.loads(crawler.spider.meta['responses'][0].body) + echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) + # The noconnect mode isn't supported by the current mitmproxy, it returns + # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all, + # and it's not clear what behavior is intended by Scrapy and by mitmproxy here. + # https://github.com/mitmproxy/mitmproxy/issues/848 may be related. + # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera, + # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related. + + @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') + @defer.inlineCallbacks + def test_https_noconnect(self): + proxy = os.environ['https_proxy'] + os.environ['https_proxy'] = proxy + '?noconnect' + crawler = get_crawler(SimpleSpider) + with LogCapture() as l: + yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) + self._assert_got_response_code(200, l) + + @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests') @defer.inlineCallbacks def test_https_noconnect_auth_error(self): os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index f89042b3d..d5a3371ab 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypes from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers + class ResponseTypesTest(unittest.TestCase): def test_from_filename(self): diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index cd7480e33..27d79437b 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -5,7 +5,7 @@ from twisted.trial import unittest def reppy_available(): # check if reppy parser is installed try: - from reppy.robots import Robots + from reppy.robots import Robots # noqa: F401 except ImportError: return False return True @@ -14,19 +14,21 @@ def reppy_available(): def rerp_available(): # check if robotexclusionrulesparser is installed try: - from robotexclusionrulesparser import RobotExclusionRulesParser + from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401 except ImportError: return False return True + def protego_available(): # check if protego parser is installed try: - from protego import Protego + from protego import Protego # noqa: F401 except ImportError: return False return True + class BaseRobotParserTest: def _setUp(self, parser_cls): self.parser_cls = parser_cls diff --git a/tests/test_selector.py b/tests/test_selector.py index b2565dd78..09c2546fb 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,9 +1,9 @@ -import warnings import weakref + from twisted.trial import unittest + from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import Selector -from lxml import etree class SelectorTestCase(unittest.TestCase): diff --git a/tests/test_spider.py b/tests/test_spider.py index c0fccfdd6..317a27076 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -15,7 +15,6 @@ from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler @@ -385,6 +384,14 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) + def test_start_url(self): + spider = self.spider_class("example.com") + spider.start_url = 'https://www.example.com' + + with self.assertRaisesRegex(AttributeError, + r'^Crawling could not start.*$'): + list(spider.start_requests()) + class SitemapSpiderTest(SpiderTest): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 106da798c..d8be6e277 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py index 35b71870a..dbd1fb123 100644 --- a/tests/test_spiderloader/test_spiders/nested/spider4.py +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider4(Spider): name = "spider4" allowed_domains = ['spider4.com'] diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py index 75a90794e..af679dbd6 100644 --- a/tests/test_spiderloader/test_spiders/spider0.py +++ b/tests/test_spiderloader/test_spiders/spider0.py @@ -1,4 +1,5 @@ from scrapy.spiders import Spider + class Spider0(Spider): allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py index 76efddc7f..6b4317a90 100644 --- a/tests/test_spiderloader/test_spiders/spider1.py +++ b/tests/test_spiderloader/test_spiders/spider1.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider1(Spider): name = "spider1" allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py index 0badd8437..352601863 100644 --- a/tests/test_spiderloader/test_spiders/spider2.py +++ b/tests/test_spiderloader/test_spiders/spider2.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider2(Spider): name = "spider2" allowed_domains = ["scrapy2.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py index d406f2d4f..84998ba35 100644 --- a/tests/test_spiderloader/test_spiders/spider3.py +++ b/tests/test_spiderloader/test_spiders/spider3.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider3(Spider): name = "spider3" allowed_domains = ['spider3.com'] diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 7e4af0d4c..b97d9b675 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -9,6 +9,7 @@ from scrapy.spidermiddlewares.offsite import URLWarning from scrapy.utils.test import get_crawler import warnings + class TestOffsiteMiddleware(TestCase): def setUp(self): @@ -53,6 +54,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) + class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spider(self): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 6f8727a15..5b7b5e7aa 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -6,7 +6,6 @@ from twisted.internet import defer from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import MockServerSpider class LogExceptionMiddleware: @@ -34,6 +33,7 @@ class RecoverySpider(Spider): if not response.meta.get('dont_fail'): raise TabError() + class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) @@ -50,6 +50,7 @@ class FailProcessSpiderInputMiddleware: spider.logger.info('Middleware: will raise IndexError') raise IndexError() + class ProcessSpiderInputSpiderWithoutErrback(Spider): name = 'ProcessSpiderInputSpiderWithoutErrback' custom_settings = { @@ -177,6 +178,7 @@ class GeneratorRecoverMiddleware: spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} + class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass @@ -247,6 +249,7 @@ class NotGeneratorRecoverMiddleware: spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] + class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): pass diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 21439c20e..2be6a1cd5 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -2,7 +2,6 @@ from six.moves.urllib.parse import urlparse from unittest import TestCase import warnings -from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider @@ -349,6 +348,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware): ] + # --- Tests using Request meta dict to set policy class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} @@ -518,14 +518,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): """ The empty string means "no-referrer-when-downgrade" diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 3ded5c027..d5fcf2f7f 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -7,16 +7,20 @@ from scrapy.http import Request from scrapy.loader import ItemLoader from scrapy.selector import Selector + class TestItem(Item): name = Field() + def _test_procesor(x): return x + x + class TestLoader(ItemLoader): default_item_class = TestItem name_out = staticmethod(_test_procesor) + def nonserializable_object_test(self): q = self.queue() try: @@ -35,6 +39,7 @@ def nonserializable_object_test(self): sel = Selector(text='<html><body><p>some text</p></body></html>') self.assertRaises(ValueError, q.push, sel) + class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): chunksize = 100000 @@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test + class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 + class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 2 + class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 3 + class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 4 @@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): self.assertEqual(r.url, r2.url) assert r2.meta['request'] is r2 + class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 1 + class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 2 + class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 3 + class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 4 diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index c2211848c..380c41367 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -14,6 +14,7 @@ try: except ImportError: ipy = False + class UtilsConsoleTestCase(unittest.TestCase): def test_get_shell_embed_func(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 003bb9b02..d642ed3ed 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -33,14 +33,23 @@ class MustbeDeferredTest(unittest.TestCase): steps.append(2) # add another value, that should be catched by assertEqual return dfd + def cb1(value, arg1, arg2): return "(cb1 %s %s %s)" % (value, arg1, arg2) + + def cb2(value, arg1, arg2): return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + + def cb3(value, arg1, arg2): return "(cb3 %s %s %s)" % (value, arg1, arg2) + + def cb_fail(value, arg1, arg2): return Failure(TypeError()) + + def eb1(failure, arg1, arg2): return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2524153ea..f9af4bf87 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -2,6 +2,7 @@ import unittest from scrapy.utils.http import decode_chunked_transfer + class ChunkedTest(unittest.TestCase): def test_decode_chunked_transfer(self): diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 4f9f7a370..2c3965bbc 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -4,6 +4,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached + class HttpobjUtilsTest(unittest.TestCase): def test_urlparse_cached(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 2d845697e..f16ef8110 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -235,6 +235,7 @@ class LxmlXmliterTestCase(XmliterTestCase): i = self.xmliter(42, 'product') self.assertRaises(TypeError, next, i) + class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index a94398796..2d27d4b81 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -5,10 +5,11 @@ import unittest from itertools import count import platform import six +from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) __doctests__ = ['scrapy.utils.python'] @@ -21,8 +22,12 @@ class MutableChainTest(unittest.TestCase): m.extend([7, 8]) m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) - self.assertEqual(m.next(), 1) - self.assertEqual(m.__next__(), 2) + self.assertEqual(m.__next__(), 1) + with catch_warnings(record=True) as warnings: + self.assertEqual(m.next(), 2) + self.assertEqual(len(warnings), 1) + self.assertIn('scrapy.utils.python.MutableChain.__next__', + str(warnings[0].message)) self.assertEqual(list(m), list(range(3, 13))) @@ -163,33 +168,6 @@ class UtilsPythonTestCase(unittest.TestCase): gc.collect() self.assertFalse(len(wk._weakdict)) - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict(self): - d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} - d2 = stringify_dict(d, keys_only=False) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) - - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict_tuples(self): - tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')] - d = dict(tuples) - d2 = stringify_dict(tuples, keys_only=False) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) - - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict_keys_only(self): - d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} - d2 = stringify_dict(d) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - def test_get_func_args(self): def f1(a, b, c): pass diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 92cd16de7..06d9c004c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,8 +1,4 @@ -# -*- coding: utf-8 -*- import unittest -import sys - -import six from scrapy.http import Request, FormRequest from scrapy.spiders import Spider diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 625a32048..3da95b95a 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -4,6 +4,7 @@ from scrapy.http import Request from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ request_authenticate, request_httprepr + class UtilsRequestTest(unittest.TestCase): def test_request_fingerprint(self): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 62edd420d..16b7c5c68 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) + class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 716bb44eb..db323ab31 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -2,6 +2,7 @@ import unittest from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots + class SitemapTest(unittest.TestCase): def test_sitemap(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index d9de1ce77..edeeacc80 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -9,12 +9,15 @@ from scrapy.spiders import CrawlSpider class MyBaseSpider(CrawlSpider): pass # abstract spider + class MySpider1(MyBaseSpider): name = 'myspider1' + class MySpider2(MyBaseSpider): name = 'myspider2' + class UtilsSpidersTestCase(unittest.TestCase): def test_iterate_spider_output(self): diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index e6588055c..c7bcaf88b 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,13 +1,9 @@ # -*- coding: utf-8 -*- import unittest -import six -from six.moves.urllib.parse import urlparse - from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, - parse_url, strip_url) + add_http_if_no_scheme, guess_scheme, strip_url) __doctests__ = ['scrapy.utils.url'] @@ -187,6 +183,7 @@ class AddHttpIfNoScheme(unittest.TestCase): class GuessSchemeTest(unittest.TestCase): pass + def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) @@ -195,6 +192,7 @@ def create_guess_scheme_t(args): args[0], url, args[1]) return do_expected + def create_skipped_scheme_t(args): def do_expected(self): raise unittest.SkipTest(args[2]) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a81946490..7b015ff8d 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -78,26 +78,6 @@ class ParseUrlTestCase(unittest.TestCase): to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEqual(client._parse(url), test, url) - def test_externalUnicodeInterference(self): - """ - L{client._parse} should return C{str} for the scheme, host, and path - elements of its return tuple, even when passed an URL which has - previously been passed to L{urlparse} as a C{unicode} string. - """ - if not six.PY2: - raise unittest.SkipTest( - "Applies only to Py2, as urls can be ONLY unicode on Py3") - badInput = u'http://example.com/path' - goodInput = badInput.encode('ascii') - self._parse(badInput) # cache badInput in urlparse_cached - scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, str)) - self.assertTrue(isinstance(netloc, str)) - self.assertTrue(isinstance(host, str)) - self.assertTrue(isinstance(path, str)) - self.assertTrue(isinstance(port, int)) - - class ScrapyHTTPPageGetterTests(unittest.TestCase):