diff --git a/.bandit.yml b/.bandit.yml index 00554587a..243379b0b 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -1,13 +1,15 @@ skips: - B101 - B105 +- B301 - B303 - B306 - B307 - B311 - B320 - B321 -- B402 +- B402 # https://github.com/scrapy/scrapy/issues/4180 +- B403 - B404 - B406 - B410 diff --git a/.travis.yml b/.travis.yml index c9c64e990..c870934e1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,9 @@ matrix: - env: TOXENV=flake8 python: 3.8 - env: TOXENV=pypy3 - python: 3.5 - env: TOXENV=py35 python: 3.5 - - env: TOXENV=py35-pinned + - env: TOXENV=pinned python: 3.5 - env: TOXENV=py36 python: 3.6 @@ -23,7 +22,7 @@ matrix: python: 3.7 - env: TOXENV=py38 python: 3.8 - - env: TOXENV=py38-extra-deps + - env: TOXENV=extra-deps python: 3.8 - env: TOXENV=docs python: 3.8 diff --git a/conftest.py b/conftest.py index d5d61ddd3..d37c22436 100644 --- a/conftest.py +++ b/conftest.py @@ -1,18 +1,23 @@ -import six +from pathlib import Path + import pytest +def _py_files(folder): + return (str(p) for p in Path(folder).rglob('*.py')) + + collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", + # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + *_py_files("tests/CrawlerProcess") ] - -if six.PY3: - for line in open('tests/py3-ignores.txt'): - file_path = line.strip() - if file_path and file_path[0] != '#': - collect_ignore.append(file_path) +for line in open('tests/ignores.txt'): + file_path = line.strip() + if file_path and file_path[0] != '#': + collect_ignore.append(file_path) @pytest.fixture() diff --git a/docs/conf.py b/docs/conf.py index 6ec4582b1..ed56c5cd1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,6 +12,7 @@ # serve to show the default. import sys +from datetime import datetime from os import path # If your extensions are in another directory, add it here. If the directory @@ -27,6 +28,7 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ + 'hoverxref.extension', 'notfound.extension', 'scrapydocs', 'sphinx.ext.autodoc', @@ -48,8 +50,8 @@ source_suffix = '.rst' master_doc = 'index' # General information about the project. -project = u'Scrapy' -copyright = u'2008–2018, Scrapy developers' +project = 'Scrapy' +copyright = '2008–{}, Scrapy developers'.format(datetime.now().year) # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -193,8 +195,8 @@ htmlhelp_basename = 'Scrapydoc' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). latex_documents = [ - ('index', 'Scrapy.tex', u'Scrapy Documentation', - u'Scrapy developers', 'manual'), + ('index', 'Scrapy.tex', 'Scrapy Documentation', + 'Scrapy developers', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -274,6 +276,16 @@ coverage_ignore_pyobjects = [ # ------------------------------------- intersphinx_mapping = { + 'coverage': ('https://coverage.readthedocs.io/en/stable', None), + 'pytest': ('https://docs.pytest.org/en/latest', None), 'python': ('https://docs.python.org/3', None), - 'sphinx': ('https://www.sphinx-doc.org/en/stable', None), + 'sphinx': ('https://www.sphinx-doc.org/en/master', None), + 'tox': ('https://tox.readthedocs.io/en/latest', None), + 'twisted': ('https://twistedmatrix.com/documents/current', None), } + + +# Options for sphinx-hoverxref options +# ------------------------------------ + +hoverxref_auto_ref = True diff --git a/docs/contributing.rst b/docs/contributing.rst index f084bd23d..3aebb3d50 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -194,61 +194,54 @@ documentation instead of duplicating the docstring in files within the Tests ===== -Tests are implemented using the `Twisted unit-testing framework`_, running -tests requires `tox`_. +Tests are implemented using the :doc:`Twisted unit-testing framework +`. Running tests requires +:doc:`tox `. .. _running-tests: Running tests ------------- -Make sure you have a recent enough `tox`_ installation: +To run all tests:: - ``tox --version`` - -If your version is older than 1.7.0, please update it first: - - ``pip install -U tox`` - -To run all tests go to the root directory of Scrapy source code and run: - - ``tox`` + tox To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` -To run the tests on a specific tox_ environment, use ``-e `` with an -environment name from ``tox.ini``. For example, to run the tests with Python -3.6 use:: +To run the tests on a specific :doc:`tox ` environment, use +``-e `` with an environment name from ``tox.ini``. For example, to run +the tests with Python 3.6 use:: tox -e py36 -You can also specify a comma-separated list of environmets, and use `tox’s -parallel mode`_ to run the tests on multiple environments in parallel:: +You can also specify a comma-separated list of environmets, and use :ref:`tox’s +parallel mode ` to run the tests on multiple environments in +parallel:: - tox -e py27,py36 -p auto + tox -e py36,py38 -p auto -To pass command-line options to pytest_, add them after ``--`` in your call to -tox_. Using ``--`` overrides the default positional arguments defined in -``tox.ini``, so you must include those default positional arguments -(``scrapy tests``) after ``--`` as well:: +To pass command-line options to :doc:`pytest `, add them after +``--`` in your call to :doc:`tox `. Using ``--`` overrides the +default positional arguments defined in ``tox.ini``, so you must include those +default positional arguments (``scrapy tests``) after ``--`` as well:: tox -- scrapy tests -x # stop after first failure You can also use the `pytest-xdist`_ plugin. For example, to run all tests on -the Python 3.6 tox_ environment using all your CPU cores:: +the Python 3.6 :doc:`tox ` environment using all your CPU cores:: tox -e py36 -- scrapy tests -n auto -To see coverage report install `coverage`_ (``pip install coverage``) and run: +To see coverage report install :doc:`coverage ` +(``pip install coverage``) and run: ``coverage report`` see output of ``coverage --help`` for more options like html or xml report. -.. _coverage: https://pypi.python.org/pypi/coverage - Writing tests ------------- @@ -269,13 +262,9 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users .. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/en/articles/creating-a-pull-request -.. _pytest: https://docs.pytest.org/en/latest/usage.html -.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html -.. _tox: https://pypi.python.org/pypi/tox -.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode +.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 8b2fef065..01986b594 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination:: def parse(self, response): for quote in response.css('div.quote'): yield { - 'text': quote.css('span.text::text').get(), 'author': quote.xpath('span/small/text()').get(), + 'text': quote.css('span.text::text').get(), } next_page = response.css('li.next a::attr("href")').get() diff --git a/docs/requirements.txt b/docs/requirements.txt index f9db85146..773b92cea 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ Sphinx>=2.1 +sphinx-hoverxref sphinx-notfound-page sphinx_rtd_theme diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 7c8c40b5f..1c461a511 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -273,5 +273,3 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. - -.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 2effe94dc..ae25dfa2f 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -166,11 +166,10 @@ for concurrency. For more information about asynchronous programming and Twisted see these links: -* `Introduction to Deferreds in Twisted`_ +* :doc:`twisted:core/howto/defer-intro` * `Twisted - hello, asynchronous programming`_ * `Twisted Introduction - Krondo`_ .. _Twisted: https://twistedmatrix.com/trac/ -.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html .. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/ .. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/ diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index a93bee06b..5b3cd7e75 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -1,3 +1,5 @@ +.. highlight:: none + .. _topics-commands: ================= @@ -66,7 +68,9 @@ structure by default, similar to this:: The directory where the ``scrapy.cfg`` file resides is known as the *project root directory*. That file contains the name of the python module that defines -the project settings. Here is an example:: +the project settings. Here is an example: + +.. code-block:: ini [settings] default = myproject.settings @@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be shared by multiple Scrapy projects, each with its own settings module. In that case, you must define one or more aliases for those settings modules -under ``[settings]`` in your ``scrapy.cfg`` file:: +under ``[settings]`` in your ``scrapy.cfg`` file: + +.. code-block:: ini [settings] default = myproject1.settings @@ -277,6 +283,8 @@ check Run contract checks. +.. skip: start + Usage examples:: $ scrapy check -l @@ -294,6 +302,8 @@ Usage examples:: [FAILED] first_spider:parse >>> Returned 92 requests, expected 0..4 +.. skip: end + .. command:: list list @@ -481,6 +491,8 @@ Supported options: * ``--verbose`` or ``-v``: display information for each depth level +.. skip: start + Usage example:: $ scrapy parse http://www.example.com/ -c parse_item @@ -495,6 +507,8 @@ Usage example:: # Requests ----------------------------------------------------------------- [] +.. skip: end + .. command:: settings @@ -573,7 +587,9 @@ Default: ``''`` (empty string) A module to use for looking up custom Scrapy commands. This is used to add custom commands for your Scrapy project. -Example:: +Example: + +.. code-block:: python COMMANDS_MODULE = 'mybot.commands' @@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a ``scrapy.commands`` section in the entry points of the library ``setup.py`` file. -The following example adds ``my_command`` command:: +The following example adds ``my_command`` command: + +.. skip: next + +.. code-block:: python from setuptools import setup, find_packages diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 0aaad0c77..4b2588518 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the of the spider at the method level. It has the advantage of being flexible and simple to use, but does not allow debugging code inside a method. +.. highlight:: none + +.. skip: start + In order to see the item scraped from a specific url:: $ scrapy parse --spider=myspider -c parse_item -d 2 @@ -85,6 +89,8 @@ using:: $ scrapy parse --spider=myspider -d 3 'http://example.com/page1' +.. skip: end + Scrapy Shell ============ @@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides showing the response received and the output. How to debug the situation when ``parse_details`` sometimes receives no item? +.. highlight:: python + Fortunately, the :command:`shell` is your bread and butter in this case (see :ref:`topics-shell-inspect-response`):: diff --git a/docs/topics/email.rst b/docs/topics/email.rst index 12eedf2cd..72bf52227 100644 --- a/docs/topics/email.rst +++ b/docs/topics/email.rst @@ -9,13 +9,13 @@ Sending e-mail Although Python makes sending e-mails relatively easy via the `smtplib`_ library, Scrapy provides its own facility for sending e-mails which is very -easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid -interfering with the non-blocking IO of the crawler. It also provides a -simple API for sending attachments and it's very easy to configure, with a few -:ref:`settings `. +easy to use and it's implemented using :doc:`Twisted non-blocking IO +`, to avoid interfering with the non-blocking +IO of the crawler. It also provides a simple API for sending attachments and +it's very easy to configure, with a few :ref:`settings +`. .. _smtplib: https://docs.python.org/2/library/smtplib.html -.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html Quick example ============= @@ -39,7 +39,8 @@ MailSender class reference ========================== MailSender is the preferred class to use for sending emails from Scrapy, as it -uses `Twisted non-blocking IO`_, like the rest of the framework. +uses :doc:`Twisted non-blocking IO `, like the +rest of the framework. .. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index fae18200a..cdc4953c2 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following This method is called for every item pipeline component. :meth:`process_item` must either: return a dict with data, return an :class:`~scrapy.item.Item` - (or any descendant class) object, return a `Twisted Deferred`_ or raise + (or any descendant class) object, return a + :class:`~twisted.internet.defer.Deferred` or raise :exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer processed by further pipeline components. @@ -67,8 +68,6 @@ Additionally, they may also implement the following methods: :type crawler: :class:`~scrapy.crawler.Crawler` object -.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html - Item pipeline example ===================== @@ -166,7 +165,8 @@ method and how to clean up the resources properly.:: Take screenshot of item ----------------------- -This example demonstrates how to return Deferred_ from :meth:`process_item` method. +This example demonstrates how to return a +:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method. It uses Splash_ to render screenshot of item url. Pipeline makes request to locally running instance of Splash_. After request is downloaded and Deferred callback fires, it saves item to a file and adds filename to an item. @@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite return item .. _Splash: https://splash.readthedocs.io/en/stable/ -.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html Duplicates filter ----------------- diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 12a5e5c60..de3f38023 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -142,20 +142,6 @@ accept one (and only one) positional argument, which will be an iterable. containing the collected values (for that field). The result of the output processors is the value that will be finally assigned to the item. -If you want to use a plain function as a processor, make sure it receives -``self`` as the first argument:: - - def lowercase_processor(self, values): - for v in values: - yield v.lower() - - class MyItemLoader(ItemLoader): - name_in = lowercase_processor - -This is because whenever a function is assigned as a class variable, it becomes -a method and would be passed the instance as the the first argument when being -called. See `this answer on stackoverflow`_ for more details. - The other thing you need to keep in mind is that the values returned by input processors are collected internally (in lists) and then passed to output processors to populate the fields. @@ -163,7 +149,7 @@ processors to populate the fields. Last, but not least, Scrapy comes with some :ref:`commonly used processors ` built-in for convenience. -.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635 + Declaring Item Loaders ====================== @@ -491,6 +477,8 @@ ItemLoader objects .. attribute:: item The :class:`~scrapy.item.Item` object being parsed by this Item Loader. + This is mostly used as a property so when attempting to override this + value, you may want to check out :attr:`default_item_class` first. .. attribute:: context diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 431cc6027..206e7cfa5 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline: * ``success`` is a boolean which is ``True`` if the image was downloaded successfully or ``False`` if it failed for some reason - * ``file_info_or_error`` is a dict containing the following keys (if success - is ``True``) or a `Twisted Failure`_ if there was a problem. + * ``file_info_or_error`` is a dict containing the following keys (if + success is ``True``) or a :exc:`~twisted.python.failure.Failure` if + there was a problem. * ``url`` - the url where the file was downloaded from. This is the url of the request returned from the :meth:`~get_media_requests` @@ -577,5 +578,4 @@ above:: item['image_paths'] = image_paths return item -.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html .. _MD5 hash: https://en.wikipedia.org/wiki/MD5 diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index a6d4f0d6d..e3e8fdc72 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -.. seealso:: `Twisted Reactor Overview`_. +.. seealso:: :doc:`twisted:core/howto/reactor-basics` .. _run-multiple-spiders: @@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting .. _ProxyMesh: https://proxymesh.com/ .. _Google cache: http://www.googleguide.com/cached_pages.html .. _testspiders: https://github.com/scrapinghub/testspiders -.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html .. _Crawlera: https://scrapinghub.com/crawlera .. _scrapoxy: https://scrapoxy.io/ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ee37f648e..4cf367d96 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -121,8 +121,8 @@ Request objects :param errback: a function that will be called if any exception was raised while processing the request. This includes pages that failed - with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance - as first parameter. + with 404 HTTP errors and such. It receives a + :exc:`~twisted.python.failure.Failure` as first parameter. For more information, see :ref:`topics-request-response-ref-errbacks` below. :type errback: callable @@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing The errback of a request is a function that will be called when an exception is raise while processing it. -It receives a `Twisted Failure`_ instance as first parameter and can be -used to track connection establishment timeouts, DNS errors etc. +It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can +be used to track connection establishment timeouts, DNS errors etc. Here's an example spider logging all errors and catching some specific errors if needed:: @@ -816,5 +816,4 @@ XmlResponse objects adds encoding auto-discovering support by looking into the XML declaration line. See :attr:`TextResponse.encoding`. -.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html .. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241 diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ff07b9d55..3f29aa323 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti Deferred signal handlers ======================== -Some signals support returning `Twisted deferreds`_ from their handlers, see -the :ref:`topics-signals-ref` below to know which ones. +Some signals support returning :class:`~twisted.internet.defer.Deferred` +objects from their handlers, see the :ref:`topics-signals-ref` below to know +which ones. -.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html .. _topics-signals-ref: @@ -155,8 +155,8 @@ item_error :param spider: the spider which raised the exception :type spider: :class:`~scrapy.spiders.Spider` object - :param failure: the exception raised as a Twisted `Failure`_ object - :type failure: `Failure`_ object + :param failure: the exception raised + :type failure: twisted.python.failure.Failure spider_closed ------------- @@ -236,8 +236,8 @@ spider_error This signal does not support returning deferreds from their handlers. - :param failure: the exception raised as a Twisted `Failure`_ object - :type failure: `Failure`_ object + :param failure: the exception raised + :type failure: twisted.python.failure.Failure :param response: the response being processed when the exception was raised :type response: :class:`~scrapy.http.Response` object @@ -333,5 +333,3 @@ response_downloaded :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spiders.Spider` object - -.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 3bef20bf3..da7a0022b 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -from __future__ import print_function from time import time from collections import deque from twisted.web.server import Site, NOT_DONE_YET diff --git a/pytest.ini b/pytest.ini index 529ad5d27..f088e10ef 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,8 +8,6 @@ addopts = --ignore=docs/_ext --ignore=docs/conf.py --ignore=docs/news.rst - --ignore=docs/topics/commands.rst - --ignore=docs/topics/debug.rst --ignore=docs/topics/developer-tools.rst --ignore=docs/topics/dynamic-content.rst --ignore=docs/topics/items.rst @@ -22,250 +20,230 @@ addopts = --ignore=docs/utils twisted = 1 flake8-ignore = + # Files that are only meant to provide top-level imports are expected not + # to use any of their imports: + scrapy/core/downloader/handlers/http.py F401 + scrapy/http/__init__.py F401 + # Issues pending a review: # extras - extras/qps-bench-server.py E261 E501 - extras/qpsclient.py E501 E261 E501 + extras/qps-bench-server.py E501 + extras/qpsclient.py E501 E501 # scrapy/commands scrapy/commands/__init__.py E128 E501 - scrapy/commands/check.py F401 E501 + scrapy/commands/check.py E501 scrapy/commands/crawl.py E501 scrapy/commands/edit.py E501 - scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731 + scrapy/commands/fetch.py E401 E501 E128 E731 scrapy/commands/genspider.py E128 E501 E502 - scrapy/commands/list.py E302 scrapy/commands/parse.py E128 E501 E731 E226 scrapy/commands/runspider.py E501 - scrapy/commands/settings.py E302 E128 + scrapy/commands/settings.py E128 scrapy/commands/shell.py E128 E501 E502 - scrapy/commands/startproject.py E502 E127 E501 E128 + scrapy/commands/startproject.py E127 E501 E128 scrapy/commands/version.py E501 E128 - scrapy/commands/view.py F401 E302 # scrapy/contracts scrapy/contracts/__init__.py E501 W504 - scrapy/contracts/default.py E502 E128 + scrapy/contracts/default.py E128 # scrapy/core - scrapy/core/engine.py E261 E501 E128 E127 E306 E502 + scrapy/core/engine.py E501 E128 E127 E306 E502 scrapy/core/scheduler.py E501 - scrapy/core/scraper.py E501 E306 E261 E128 W504 - scrapy/core/spidermw.py E501 E731 E502 E126 E226 - scrapy/core/downloader/__init__.py F401 E501 + scrapy/core/scraper.py E501 E306 E128 W504 + scrapy/core/spidermw.py E501 E731 E126 E226 + scrapy/core/downloader/__init__.py E501 scrapy/core/downloader/contextfactory.py E501 E128 E126 scrapy/core/downloader/middleware.py E501 E502 scrapy/core/downloader/tls.py E501 E305 E241 - scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226 + scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226 scrapy/core/downloader/handlers/__init__.py E501 scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127 - scrapy/core/downloader/handlers/http.py F401 scrapy/core/downloader/handlers/http10.py E501 scrapy/core/downloader/handlers/http11.py E501 - scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126 + scrapy/core/downloader/handlers/s3.py E501 E128 E126 # scrapy/downloadermiddlewares - scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226 + scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226 scrapy/downloadermiddlewares/decompression.py E501 scrapy/downloadermiddlewares/defaultheaders.py E501 scrapy/downloadermiddlewares/httpcache.py E501 E126 - scrapy/downloadermiddlewares/httpcompression.py E502 E128 + scrapy/downloadermiddlewares/httpcompression.py E501 E128 scrapy/downloadermiddlewares/httpproxy.py E501 scrapy/downloadermiddlewares/redirect.py E501 W504 scrapy/downloadermiddlewares/retry.py E501 E126 - scrapy/downloadermiddlewares/robotstxt.py F401 E501 + scrapy/downloadermiddlewares/robotstxt.py E501 scrapy/downloadermiddlewares/stats.py E501 # scrapy/extensions - scrapy/extensions/closespider.py E501 E502 E128 E123 - scrapy/extensions/corestats.py E302 E501 + scrapy/extensions/closespider.py E501 E128 E123 + scrapy/extensions/corestats.py E501 scrapy/extensions/feedexport.py E128 E501 - scrapy/extensions/httpcache.py E128 E501 E303 F401 + scrapy/extensions/httpcache.py E128 E501 E303 scrapy/extensions/memdebug.py E501 - scrapy/extensions/spiderstate.py E302 E501 + scrapy/extensions/spiderstate.py E501 scrapy/extensions/telnet.py E501 W504 scrapy/extensions/throttle.py E501 # scrapy/http - scrapy/http/__init__.py F401 scrapy/http/common.py E501 scrapy/http/cookies.py E501 scrapy/http/request/__init__.py E501 scrapy/http/request/form.py E501 E123 scrapy/http/request/json_request.py E501 scrapy/http/response/__init__.py E501 E128 W293 W291 - scrapy/http/response/html.py E302 scrapy/http/response/text.py E501 W293 E128 E124 - scrapy/http/response/xml.py E302 # scrapy/linkextractors - scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401 + scrapy/linkextractors/__init__.py E731 E501 E402 scrapy/linkextractors/lxmlhtml.py E501 E731 E226 # scrapy/loader - scrapy/loader/__init__.py E501 E502 E128 - scrapy/loader/common.py E302 + scrapy/loader/__init__.py E501 E128 scrapy/loader/processors.py E501 # scrapy/pipelines - scrapy/pipelines/__init__.py E302 scrapy/pipelines/files.py E116 E501 E266 scrapy/pipelines/images.py E265 E501 scrapy/pipelines/media.py E125 E501 E266 # scrapy/selector - scrapy/selector/__init__.py F403 F401 - scrapy/selector/unified.py F401 E501 E111 + scrapy/selector/__init__.py F403 + scrapy/selector/unified.py E501 E111 # scrapy/settings scrapy/settings/__init__.py E501 - scrapy/settings/default_settings.py E501 E261 E114 E116 E226 + scrapy/settings/default_settings.py E501 E114 E116 E226 scrapy/settings/deprecated.py E501 # scrapy/spidermiddlewares scrapy/spidermiddlewares/httperror.py E501 scrapy/spidermiddlewares/offsite.py E501 - scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504 + scrapy/spidermiddlewares/referer.py E501 E129 W503 W504 scrapy/spidermiddlewares/urllength.py E501 # scrapy/spiders - scrapy/spiders/__init__.py F401 E501 E402 + scrapy/spiders/__init__.py E501 E402 scrapy/spiders/crawl.py E501 - scrapy/spiders/feed.py E501 E261 + scrapy/spiders/feed.py E501 scrapy/spiders/sitemap.py E501 # scrapy/utils scrapy/utils/benchserver.py E501 - scrapy/utils/boto.py F401 - scrapy/utils/conf.py E402 E502 E501 - scrapy/utils/console.py E302 E261 F401 E306 E305 - scrapy/utils/curl.py F401 + scrapy/utils/conf.py E402 E501 + scrapy/utils/console.py E306 E305 scrapy/utils/datatypes.py E501 E226 - scrapy/utils/decorators.py E501 E302 - scrapy/utils/defer.py E501 E302 E128 + scrapy/utils/decorators.py E501 + scrapy/utils/defer.py E501 E128 scrapy/utils/deprecate.py E128 E501 E127 E502 - scrapy/utils/display.py E302 - scrapy/utils/engine.py F401 E261 E302 - scrapy/utils/ftp.py E302 - scrapy/utils/gz.py E305 E501 E302 W504 - scrapy/utils/http.py F403 F401 E226 - scrapy/utils/httpobj.py E302 E501 + scrapy/utils/gz.py E305 E501 W504 + scrapy/utils/http.py F403 E226 + scrapy/utils/httpobj.py E501 scrapy/utils/iterators.py E501 E701 - scrapy/utils/job.py E302 scrapy/utils/log.py E128 W503 - scrapy/utils/markup.py F403 F401 W292 + scrapy/utils/markup.py F403 W292 scrapy/utils/misc.py E501 E226 - scrapy/utils/multipart.py F403 F401 W292 + scrapy/utils/multipart.py F403 W292 scrapy/utils/project.py E501 - scrapy/utils/python.py E501 E302 - scrapy/utils/reactor.py E302 E226 + scrapy/utils/python.py E501 + scrapy/utils/reactor.py E226 scrapy/utils/reqser.py E501 - scrapy/utils/request.py E302 E127 E501 - scrapy/utils/response.py E501 E302 E128 + scrapy/utils/request.py E127 E501 + scrapy/utils/response.py E501 E128 scrapy/utils/signal.py E501 E128 scrapy/utils/sitemap.py E501 - scrapy/utils/spider.py E271 E302 E501 + scrapy/utils/spider.py E271 E501 scrapy/utils/ssl.py E501 - scrapy/utils/template.py E302 - scrapy/utils/test.py E302 E501 - scrapy/utils/url.py E501 F403 F401 E128 F405 + scrapy/utils/test.py E501 + scrapy/utils/url.py E501 F403 E128 F405 # scrapy scrapy/__init__.py E402 E501 scrapy/_monkeypatches.py W293 - scrapy/cmdline.py E502 E501 + scrapy/cmdline.py E501 scrapy/crawler.py E501 - scrapy/dupefilters.py E302 E501 E202 - scrapy/exceptions.py E302 E501 - scrapy/exporters.py E501 E261 E226 - scrapy/extension.py E302 - scrapy/interfaces.py E302 E501 + scrapy/dupefilters.py E501 E202 + scrapy/exceptions.py E501 + scrapy/exporters.py E501 E226 + scrapy/interfaces.py E501 scrapy/item.py E501 E128 scrapy/link.py E501 scrapy/logformatter.py E501 W293 scrapy/mail.py E402 E128 E501 E502 - scrapy/middleware.py E502 E128 E501 + scrapy/middleware.py E128 E501 scrapy/pqueues.py E501 - scrapy/resolver.py E302 scrapy/responsetypes.py E128 E501 E305 - scrapy/robotstxt.py E302 E501 + scrapy/robotstxt.py E501 scrapy/shell.py E501 scrapy/signalmanager.py E501 scrapy/spiderloader.py E225 F841 E501 E126 scrapy/squeues.py E128 scrapy/statscollectors.py E501 # tests - tests/__init__.py F401 E402 E501 - tests/mockserver.py E401 E501 E126 E123 F401 - tests/pipelines.py E302 F841 E226 - tests/spiders.py E302 E501 E127 + tests/__init__.py E402 E501 + tests/mockserver.py E401 E501 E126 E123 + tests/pipelines.py F841 E226 + tests/spiders.py E501 E127 tests/test_closespider.py E501 E127 - tests/test_command_fetch.py E501 E261 - tests/test_command_parse.py F401 E302 E501 E128 E303 E226 + tests/test_command_fetch.py E501 + tests/test_command_parse.py E501 E128 E303 E226 tests/test_command_shell.py E501 E128 - tests/test_commands.py F401 E128 E501 + tests/test_commands.py E128 E501 tests/test_contracts.py E501 E128 W293 tests/test_crawl.py E501 E741 E265 tests/test_crawler.py F841 E306 E501 - tests/test_dependencies.py E302 F841 E501 E305 - tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E126 E226 E123 + tests/test_dependencies.py F841 E501 E305 + tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123 tests/test_downloadermiddleware.py E501 - tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501 + tests/test_downloadermiddleware_ajaxcrawlable.py E501 tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126 tests/test_downloadermiddleware_decompression.py E127 tests/test_downloadermiddleware_defaultheaders.py E501 tests/test_downloadermiddleware_downloadtimeout.py E501 - tests/test_downloadermiddleware_httpcache.py E501 E302 E305 F401 - tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123 - tests/test_downloadermiddleware_httpproxy.py F401 E501 E128 + tests/test_downloadermiddleware_httpcache.py E501 E305 + tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123 + tests/test_downloadermiddleware_httpproxy.py E501 E128 tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305 - tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126 + tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126 tests/test_downloadermiddleware_robotstxt.py E501 tests/test_downloadermiddleware_stats.py E501 - tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124 - tests/test_engine.py E401 E501 E502 E128 E261 + tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124 + tests/test_engine.py E401 E501 E128 tests/test_exporters.py E501 E731 E306 E128 E124 - tests/test_extension_telnet.py F401 F841 - tests/test_feedexport.py E501 F401 F841 E241 + tests/test_extension_telnet.py F841 + tests/test_feedexport.py E501 F841 E241 tests/test_http_cookies.py E501 - tests/test_http_headers.py E302 E501 - tests/test_http_request.py F401 E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123 - tests/test_http_response.py E501 E301 E502 E128 E265 + tests/test_http_headers.py E501 + tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123 + tests/test_http_response.py E501 E301 E128 E265 tests/test_item.py E701 E128 F841 E306 tests/test_link.py E501 tests/test_linkextractors.py E501 E128 E124 - tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241 - tests/test_logformatter.py E128 E501 E122 E302 - tests/test_mail.py E302 E128 E501 E305 - tests/test_middleware.py E302 E501 E128 + tests/test_loader.py E501 E731 E303 E741 E128 E117 E241 + tests/test_logformatter.py E128 E501 E122 + tests/test_mail.py E128 E501 E305 + tests/test_middleware.py E501 E128 tests/test_pipeline_crawl.py E131 E501 E128 E126 - tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226 - tests/test_pipeline_images.py F401 F841 E501 E303 - tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502 + tests/test_pipeline_files.py E501 W293 E303 E272 E226 + tests/test_pipeline_images.py F841 E501 E303 + tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502 + tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 - tests/test_responsetypes.py E501 E302 E305 - tests/test_robotstxt_interface.py F401 E302 E501 W291 E501 + tests/test_responsetypes.py E501 E305 + tests/test_robotstxt_interface.py E501 W291 E501 tests/test_scheduler.py E501 E126 E123 - tests/test_selector.py F401 E501 E127 - tests/test_spider.py E501 F401 + tests/test_selector.py E501 E127 + tests/test_spider.py E501 tests/test_spidermiddleware.py E501 E226 tests/test_spidermiddleware_httperror.py E128 E501 E127 E121 - tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293 - tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226 - tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121 - tests/test_squeues.py E501 E302 E701 E741 + tests/test_spidermiddleware_offsite.py E501 E128 E111 W293 + tests/test_spidermiddleware_output_chain.py E501 W293 E226 + tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121 + tests/test_squeues.py E501 E701 E741 tests/test_utils_conf.py E501 E303 E128 - tests/test_utils_console.py E302 tests/test_utils_curl.py E501 tests/test_utils_datatypes.py E402 E501 E305 - tests/test_utils_defer.py E306 E261 E501 E302 F841 E226 + tests/test_utils_defer.py E306 E501 F841 E226 tests/test_utils_deprecate.py F841 E306 E501 - tests/test_utils_http.py E302 E501 E502 E128 W504 - tests/test_utils_httpobj.py E302 - tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241 + tests/test_utils_http.py E501 E128 W504 + tests/test_utils_iterators.py E501 E128 E129 E303 E241 tests/test_utils_log.py E741 E226 tests/test_utils_python.py E501 E303 E731 E701 E305 - tests/test_utils_reqser.py F401 E501 E128 - tests/test_utils_request.py E302 E501 E128 E305 + tests/test_utils_reqser.py E501 E128 + tests/test_utils_request.py E501 E128 E305 tests/test_utils_response.py E501 - tests/test_utils_signal.py E741 F841 E302 E731 E226 - tests/test_utils_sitemap.py E302 E128 E501 E124 - tests/test_utils_spider.py E261 E302 E305 + tests/test_utils_signal.py E741 F841 E731 E226 + tests/test_utils_sitemap.py E128 E501 E124 + tests/test_utils_spider.py E305 tests/test_utils_template.py E305 - tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123 + tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123 tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126 - tests/mocks/dummydbm.py E302 - tests/test_cmdline/__init__.py E502 E501 - tests/test_cmdline/extensions.py E302 - tests/test_settings/__init__.py F401 E501 E128 - tests/test_spiderloader/__init__.py E128 E501 E302 - tests/test_spiderloader/test_spiders/spider0.py E302 - tests/test_spiderloader/test_spiders/spider1.py E302 - tests/test_spiderloader/test_spiders/spider2.py E302 - tests/test_spiderloader/test_spiders/spider3.py E302 - tests/test_spiderloader/test_spiders/nested/spider4.py E302 + tests/test_cmdline/__init__.py E501 + tests/test_settings/__init__.py E501 E128 + tests/test_spiderloader/__init__.py E128 E501 tests/test_utils_misc/__init__.py E501 diff --git a/requirements-py3.txt b/requirements-py3.txt deleted file mode 100644 index 2c98e6f6d..000000000 --- a/requirements-py3.txt +++ /dev/null @@ -1,18 +0,0 @@ -parsel>=1.5.0 -PyDispatcher>=2.0.5 -Twisted>=17.9.0 -w3lib>=1.17.0 -protego>=0.1.15 - -pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'" -queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'" -cryptography>=2.0 # Earlier versions would fail to install - -# Reference versions taken from -# https://packages.ubuntu.com/xenial/python/ -# https://packages.ubuntu.com/xenial/zope/ -cssselect>=0.9.1 -lxml>=3.5.0 -service_identity>=16.0.0 -six>=1.10.0 -zope.interface>=4.1.3 diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 230e5cee3..fb8357f3c 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted') del warnings # Apply monkey patches to fix issues in external libraries -from . import _monkeypatches +from scrapy import _monkeypatches del _monkeypatches from twisted import version as _txv diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 1f8067b35..f74f89bda 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves import copyreg +import copyreg # Undo what Twisted's perspective broker adds to pickle register diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 418dc1ac9..ec78f7c91 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys import os import optparse @@ -68,7 +67,7 @@ def _pop_command_name(argv): def _print_header(settings, inproject): if inproject: - print("Scrapy %s - project: %s\n" % (scrapy.__version__, \ + print("Scrapy %s - project: %s\n" % (scrapy.__version__, settings['BOT_NAME'])) else: print("Scrapy %s - no active project\n" % scrapy.__version__) @@ -124,7 +123,7 @@ def execute(argv=None, settings=None): inproject = inside_project() cmds = _get_commands_dict(settings, inproject) cmdname = _pop_command_name(argv) - parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ + parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), conflict_handler='resolve') if not cmdname: _print_commands(settings, inproject) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 90c8d56a2..7bbe362e7 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,8 +1,7 @@ import sys import time import subprocess - -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode import scrapy from scrapy.commands import ScrapyCommand diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 3e6c11b7d..9d4437a47 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -1,6 +1,4 @@ -from __future__ import print_function import time -import sys from collections import defaultdict from unittest import TextTestRunner, TextTestResult as _TextTestResult diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index d45133e0e..0e149941d 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -1,4 +1,3 @@ -from __future__ import print_function import sys from w3lib.url import is_url @@ -8,6 +7,7 @@ from scrapy.exceptions import UsageError from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider + class Command(ScrapyCommand): requires_project = False @@ -24,12 +24,11 @@ class Command(ScrapyCommand): def add_options(self, parser): ScrapyCommand.add_options(self, parser) - parser.add_option("--spider", dest="spider", - help="use this spider") - parser.add_option("--headers", dest="headers", action="store_true", \ - help="print response HTTP headers instead of body") - parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ - default=False, help="do not handle HTTP 3xx status codes and print response as-is") + parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--headers", dest="headers", action="store_true", + help="print response HTTP headers instead of body") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index d5498bb5c..adb01fa70 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os import shutil import string diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index a255b3b94..54d7bb228 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,6 @@ -from __future__ import print_function from scrapy.commands import ScrapyCommand + class Command(ScrapyCommand): requires_project = True diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index ef8acd29c..ff6f1d8cd 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -1,4 +1,3 @@ -from __future__ import print_function import json import logging diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index bee52f06a..603bafb9f 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,9 +1,9 @@ -from __future__ import print_function import json from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings + class Command(ScrapyCommand): requires_project = False diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 3b9f6eabb..b123e5c84 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -1,4 +1,3 @@ -from __future__ import print_function import re import os import string @@ -44,8 +43,8 @@ class Command(ScrapyCommand): return False if not re.search(r'^[_a-zA-Z]\w*$', project_name): - print('Error: Project names must begin with a letter and contain'\ - ' only\nletters, numbers and underscores') + print('Error: Project names must begin with a letter and contain' + ' only\nletters, numbers and underscores') elif _module_exists(project_name): print('Error: Module %r already exists' % project_name) else: diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 8651948f7..1516c5997 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import scrapy from scrapy.commands import ScrapyCommand from scrapy.utils.versions import scrapy_components_versions diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py index 59e665016..41e77ba3b 100644 --- a/scrapy/commands/view.py +++ b/scrapy/commands/view.py @@ -1,6 +1,7 @@ -from scrapy.commands import fetch, ScrapyCommand +from scrapy.commands import fetch from scrapy.utils.response import open_in_browser + class Command(fetch.Command): def short_desc(self): diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 24f6c2e77..3002fc702 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -4,7 +4,7 @@ from scrapy.item import BaseItem from scrapy.http import Request from scrapy.exceptions import ContractFail -from . import Contract +from scrapy.contracts import Contract # contracts @@ -86,8 +86,8 @@ class ReturnsContract(Contract): else: expected = '%s..%s' % (self.min_bound, self.max_bound) - raise ContractFail("Returned %s %s, expected %s" % \ - (occurrences, self.obj_name, expected)) + raise ContractFail("Returned %s %s, expected %s" % + (occurrences, self.obj_name, expected)) class ScrapesContract(Contract): diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 949dacbc8..157dc3418 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,19 +1,16 @@ -from __future__ import absolute_import import random -import warnings from time import time from datetime import datetime from collections import deque -import six from twisted.internet import reactor, defer, task from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache from scrapy import signals -from .middleware import DownloaderMiddlewareManager -from .handlers import DownloadHandlers +from scrapy.core.downloader.middleware import DownloaderMiddlewareManager +from scrapy.core.downloader.handlers import DownloadHandlers class Slot(object): @@ -190,7 +187,7 @@ class Downloader(object): def close(self): self._slot_gc_loop.stop() - for slot in six.itervalues(self.slots): + for slot in self.slots.values(): slot.close() def _slot_gc(self, age=60): diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 89d2776ae..6e023ebcc 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory): """ Twisted-recommended context factory for web clients. - Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: - "The default is to use a BrowserLikePolicyForHTTPS, - so unless you have special requirements you can leave this as-is." + Quoting the documentation of the :class:`~twisted.web.client.Agent` class: - creatorForNetloc() is the same as BrowserLikePolicyForHTTPS - except this context factory allows setting the TLS/SSL method to use. + The default is to use a + :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you + have special requirements you can leave this as-is. - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation. + :meth:`creatorForNetloc` is the same as + :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context + factory allows setting the TLS/SSL method to use. + + The default OpenSSL method is ``TLS_METHOD`` (also called + ``SSLv23_METHOD``) which allows TLS protocol negotiation. """ def creatorForNetloc(self, hostname, port): diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 0b55d32fa..39a0b1f51 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,8 +1,9 @@ """Download handlers for different schemes""" import logging + from twisted.internet import defer -import six + from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object @@ -22,7 +23,7 @@ class DownloadHandlers(object): self._notconfigured = {} # remembers failed handlers handlers = without_none_values( crawler.settings.getwithbase('DOWNLOAD_HANDLERS')) - for scheme, clspath in six.iteritems(handlers): + for scheme, clspath in handlers.items(): self._schemes[scheme] = clspath self._load_handler(scheme, skip_lazy=True) diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index ad25beb3b..9e5020753 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -17,8 +17,8 @@ class DataURIDownloadHandler(object): respcls = responsetypes.from_mimetype(uri.media_type) resp_kwargs = {} - if (issubclass(respcls, TextResponse) and - uri.media_type.split('/')[0] == 'text'): + if (issubclass(respcls, TextResponse) + and uri.media_type.split('/')[0] == 'text'): charset = uri.media_type_parameters.get('charset') resp_kwargs['encoding'] = charset diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 39ed67a1a..aef231e82 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys: import re from io import BytesIO -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from twisted.internet import reactor from twisted.protocols.ftp import FTPClient, CommandFailed diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index ac4b867c3..52535bd8b 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -1,3 +1,4 @@ -from __future__ import absolute_import -from .http10 import HTTP10DownloadHandler -from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler +from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler +from scrapy.core.downloader.handlers.http11 import ( + HTTP11DownloadHandler as HTTPDownloadHandler, +) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7d917cb74..1212feb79 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -2,10 +2,10 @@ import re import logging +import warnings from io import BytesIO from time import time -import warnings -from six.moves.urllib.parse import urldefrag +from urllib.parse import urldefrag from zope.interface import implementer from twisted.internet import defer, reactor, protocol @@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse @@ -285,6 +286,12 @@ class ScrapyAgent(object): scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams + if omitConnectTunnel: + warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. " + "If you use Crawlera, it doesn't require this mode anymore, " + "so you should update scrapy-crawlera to 1.3.0+ " + "and remove '?noconnect' from the Crawlera URL.", + ScrapyDeprecationWarning) if scheme == b'https' and not omitConnectTunnel: proxyAuth = request.headers.get(b'Proxy-Authorization', None) proxyConf = (proxyHost, proxyPort, proxyAuth) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d8bbdd326..d6fbd54ee 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,9 +1,9 @@ -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.boto import is_botocore -from .http import HTTPDownloadHandler +from scrapy.core.downloader.handlers.http import HTTPDownloadHandler def _get_boto_connection(): @@ -21,7 +21,7 @@ def _get_boto_connection(): return http_request.headers try: - import boto.auth + import boto.auth # noqa: F401 except ImportError: _S3Connection = _v19_S3Connection else: @@ -32,8 +32,8 @@ def _get_boto_connection(): class S3DownloadHandler(object): - def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ - httpdownloadhandler=HTTPDownloadHandler, **kw): + def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, + httpdownloadhandler=HTTPDownloadHandler, **kw): if not aws_access_key_id: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 7a6a4dfac..38608a429 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -3,8 +3,6 @@ Downloader Middleware manager See documentation in docs/topics/downloader-middleware.rst """ -import six - from twisted.internet import defer from scrapy.exceptions import _InvalidOutput @@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) + (method.__self__.__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) defer.returnValue((yield download_func(request=request, spider=spider))) @@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, response=response, spider=spider) if not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response))) + (method.__self__.__class__.__name__, type(response))) if isinstance(response, Request): defer.returnValue(response) defer.returnValue(response) @@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): response = yield method(request=request, exception=exception, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response))) + (method.__self__.__class__.__name__, type(response))) if response: defer.returnValue(response) defer.returnValue(_failure) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3fe13414a..fc796e8bb 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,5 @@ from time import time -from six.moves.urllib.parse import urlparse, urlunparse, urldefrag +from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient @@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient): delimiter = b'\n' def connectionMade(self): - self.headers = Headers() # bucket for response headers + self.headers = Headers() # bucket for response headers # Method command self.sendCommand(self.factory.method, self.factory.path) @@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient): if self.factory.url.startswith(b'https'): self.transport.stopProducing() - self.factory.noPage(\ - defer.TimeoutError("Getting %s took longer than %s seconds." % \ - (self.factory.url, self.factory.timeout))) + self.factory.noPage( + defer.TimeoutError("Getting %s took longer than %s seconds." % + (self.factory.url, self.factory.timeout))) class ScrapyHTTPClientFactory(HTTPClientFactory): diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index fa913e528..829e69993 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -25,7 +25,7 @@ class Slot(object): def __init__(self, start_requests, close_if_idle, nextcall, scheduler): self.closing = False - self.inprogress = set() # requests in progress + self.inprogress = set() # requests in progress self.start_requests = iter(start_requests) self.close_if_idle = close_if_idle self.nextcall = nextcall diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 40de6b87a..b3d585cce 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -123,7 +123,7 @@ class Scraper(object): callback/errback""" assert isinstance(response, (Response, Failure)) - dfd = self._scrape2(response, request, spider) # returns spiders processed output + dfd = self._scrape2(response, request, spider) # returns spiders processed output dfd.addErrback(self.handle_spider_error, request, response, spider) dfd.addCallback(self.handle_spider_output, request, response, spider) return dfd @@ -231,9 +231,9 @@ class Scraper(object): signal=signals.item_dropped, item=item, response=response, spider=spider, exception=output.value) else: - logger.error('Error processing %(item)s', {'item': item}, - exc_info=failure_to_exc_info(output), - extra={'spider': spider}) + logkws = self.logformatter.error(item, ex, response, spider) + logger.log(*logformatter_adapter(logkws), extra={'spider': spider}, + exc_info=failure_to_exc_info(output)) return self.signals.send_catch_log_deferred( signal=signals.item_error, item=item, response=response, spider=spider, failure=output) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 00cee3ada..097a374bf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst """ from itertools import chain, islice -import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager @@ -37,15 +36,15 @@ class SpiderMiddlewareManager(MiddlewareManager): def scrape_response(self, scrape_func, response, request, spider): fname = lambda f: '%s.%s' % ( - six.get_method_self(f).__class__.__name__, - six.get_method_function(f).__name__) + f.__self__.__class__.__name__, + f.__func__.__name__) def process_spider_input(response): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) if result is not None: - raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \ + raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' .format(fname(method), type(result))) except _InvalidOutput: raise @@ -70,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager): elif result is None: continue else: - raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \ + raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' .format(fname(method), type(result))) return _failure @@ -104,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = evaluate_iterable(result, method_index) else: - raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ + raise _InvalidOutput('Middleware {} must return an iterable, got {}' .format(fname(method), type(result))) return chain(result, recovered) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 8868a985b..6c7eb737b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,6 +1,6 @@ -import six -import signal import logging +import pprint +import signal import warnings from twisted.internet import reactor, defer @@ -21,6 +21,7 @@ from scrapy.utils.log import ( get_scrapy_root_handler, install_scrapy_root_handler) from scrapy import signals + logger = logging.getLogger(__name__) @@ -45,7 +46,8 @@ class Crawler(object): logging.root.addHandler(handler) d = dict(overridden_settings(self.settings)) - logger.info("Overridden settings: %(settings)r", {'settings': d}) + logger.info("Overridden settings:\n%(settings)s", + {'settings': pprint.pformat(d)}) if get_scrapy_root_handler() is not None: # scrapy root handler already installed: update it with new settings @@ -110,7 +112,7 @@ class Crawler(object): class CrawlerRunner(object): """ This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup Twisted `reactor`_. + crawlers inside an already setup :mod:`~twisted.internet.reactor`. The CrawlerRunner object must be instantiated with a :class:`~scrapy.settings.Settings` object. @@ -204,7 +206,7 @@ class CrawlerRunner(object): return self._create_crawler(crawler_or_spidercls) def _create_crawler(self, spidercls): - if isinstance(spidercls, six.string_types): + if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) @@ -233,12 +235,13 @@ class CrawlerProcess(CrawlerRunner): A class to run multiple scrapy crawlers in a process simultaneously. This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a Twisted `reactor`_ and handling shutdown signals, like the - keyboard interrupt command Ctrl-C. It also configures top-level logging. + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. This utility should be a better fit than :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - Twisted `reactor`_ within your application. + :mod:`~twisted.internet.reactor` within your application. The CrawlerProcess object must be instantiated with a :class:`~scrapy.settings.Settings` object. @@ -273,9 +276,9 @@ class CrawlerProcess(CrawlerRunner): def start(self, stop_after_crawl=True): """ - This method starts a Twisted `reactor`_, adjusts its pool size to - :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based - on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache + based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. If ``stop_after_crawl`` is True, the reactor will be stopped after all crawlers have finished, using :meth:`join`. diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py index 72715dba7..7a140fcad 100644 --- a/scrapy/downloadermiddlewares/ajaxcrawl.py +++ b/scrapy/downloadermiddlewares/ajaxcrawl.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import re import logging -import six from w3lib import html from scrapy.exceptions import NotConfigured @@ -67,7 +65,9 @@ class AjaxCrawlMiddleware(object): # XXX: move it to w3lib? -_ajax_crawlable_re = re.compile(six.u(r'')) +_ajax_crawlable_re = re.compile(r'') + + def _has_ajaxcrawlable_meta(text): """ >>> _has_ajaxcrawlable_meta('') diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index 0d2b9900c..d8dabdf13 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -1,5 +1,3 @@ -import os -import six import logging from collections import defaultdict @@ -8,6 +6,7 @@ from scrapy.http import Response from scrapy.http.cookies import CookieJar from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) @@ -82,8 +81,10 @@ class CookiesMiddleware(object): def _get_request_cookies(self, jar, request): if isinstance(request.cookies, dict): - cookie_list = [{'name': k, 'value': v} for k, v in \ - six.iteritems(request.cookies)] + cookie_list = [ + {'name': k, 'value': v} + for k, v in request.cookies.items() + ] else: cookie_list = request.cookies diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py index e2d73f347..fcea38ef5 100644 --- a/scrapy/downloadermiddlewares/decompression.py +++ b/scrapy/downloadermiddlewares/decompression.py @@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive. import bz2 import gzip -from io import BytesIO -import zipfile -import tarfile import logging +import tarfile +import zipfile +from io import BytesIO from tempfile import mktemp -import six - from scrapy.responsetypes import responsetypes + logger = logging.getLogger(__name__) @@ -75,7 +74,7 @@ class DecompressionMiddleware(object): if not response.body: return response - for fmt, func in six.iteritems(self._formats): + for fmt, func in self._formats.items(): new_response = func(response) if new_response: logger.debug('Decompressed response with format: %(responsefmt)s', diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 495b103d1..4e06f8236 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,11 +1,19 @@ from email.utils import formatdate + from twisted.internet import defer -from twisted.internet.error import TimeoutError, DNSLookupError, \ - ConnectionRefusedError, ConnectionDone, ConnectError, \ - ConnectionLost, TCPTimedOutError +from twisted.internet.error import ( + ConnectError, + ConnectionDone, + ConnectionLost, + ConnectionRefusedError, + DNSLookupError, + TCPTimedOutError, + TimeoutError, +) from twisted.web.client import ResponseFailed + from scrapy import signals -from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.utils.misc import load_object diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 203dee42d..65b652953 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object): if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - respcls = responsetypes.from_args(headers=response.headers, \ - url=response.url, body=decoded_body) + respcls = responsetypes.from_args( + headers=response.headers, url=response.url, body=decoded_body + ) kwargs = dict(cls=respcls, body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2212d9688..814ce78fe 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,6 @@ import base64 -from six.moves.urllib.parse import unquote, urlunparse -from six.moves.urllib.request import getproxies, proxy_bypass -from urllib.request import _parse_proxy +from urllib.parse import unquote, urlunparse +from urllib.request import getproxies, proxy_bypass, _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index b73f864dd..77cb5aa94 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,5 +1,5 @@ import logging -from six.moves.urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string @@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured + logger = logging.getLogger(__name__) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 0bcdd3495..ea6a4cfc3 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -1,10 +1,10 @@ -from __future__ import print_function import os import logging from scrapy.utils.job import job_dir from scrapy.utils.request import referer_str, request_fingerprint + class BaseDupeFilter(object): @classmethod diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 96949bdd9..7c4bb3d00 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -7,10 +7,12 @@ new exceptions here without documenting them there. # Internal + class NotConfigured(Exception): """Indicates a missing configuration situation""" pass + class _InvalidOutput(TypeError): """ Indicates an invalid value has been returned by a middleware's processing method. @@ -18,15 +20,19 @@ class _InvalidOutput(TypeError): """ pass + # HTTP and crawling + class IgnoreRequest(Exception): """Indicates a decision was made not to process a request""" + class DontCloseSpider(Exception): """Request the spider not to be closed yet""" pass + class CloseSpider(Exception): """Raise this from callbacks to request the spider to be closed""" @@ -34,30 +40,37 @@ class CloseSpider(Exception): super(CloseSpider, self).__init__() self.reason = reason + # Items + class DropItem(Exception): """Drop item from the item pipeline""" pass + class NotSupported(Exception): """Indicates a feature or method is not supported""" pass + # Commands + class UsageError(Exception): """To indicate a command-line usage error""" def __init__(self, *a, **kw): self.print_help = kw.pop('print_help', True) super(UsageError, self).__init__(*a, **kw) + class ScrapyDeprecationWarning(Warning): """Warning category for deprecated features, since the default DeprecationWarning is silenced on Python 2.7+ """ pass + class ContractFail(AssertionError): """Error raised in case of a failing contract""" pass diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 3defafd60..5bf131312 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -6,15 +6,14 @@ import csv import io import pprint import marshal -import six -from six.moves import cPickle as pickle +import warnings +import pickle from xml.sax.saxutils import XMLGenerator from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning -import warnings __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', @@ -60,9 +59,9 @@ class BaseItemExporter(object): include_empty = self.export_empty_fields if self.fields_to_export is None: if include_empty and not isinstance(item, dict): - field_iter = six.iterkeys(item.fields) + field_iter = item.fields.keys() else: - field_iter = six.iterkeys(item) + field_iter = item.keys() else: if include_empty: field_iter = self.fields_to_export @@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter): for value in serialized_value: self._export_xml_field('value', value, depth=depth+1) self._beautify_indent(depth=depth) - elif isinstance(serialized_value, six.text_type): + elif isinstance(serialized_value, str): self.xg.characters(serialized_value) else: self.xg.characters(str(serialized_value)) @@ -200,7 +199,7 @@ class CsvItemExporter(BaseItemExporter): line_buffering=False, write_through=True, encoding=self.encoding, - newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 + newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034 ) self.csv_writer = csv.writer(self.stream, **kwargs) self._headers_not_written = True @@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter): if is_listlike(value): return [self._serialize_value(v) for v in value] encode_func = to_bytes if self.binary else to_unicode - if isinstance(value, (six.text_type, bytes)): + if isinstance(value, (str, bytes)): return encode_func(value, encoding=self.encoding) return value def _serialize_dict(self, value): - for key, val in six.iteritems(value): + for key, val in value.items(): key = to_bytes(key) if self.binary else key yield key, self._serialize_value(val) diff --git a/scrapy/extension.py b/scrapy/extension.py index e39e456fa..050b87e5f 100644 --- a/scrapy/extension.py +++ b/scrapy/extension.py @@ -6,6 +6,7 @@ See documentation in docs/topics/extensions.rst from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list + class ExtensionManager(MiddlewareManager): component_name = 'extension' diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py index 9ccf356ec..afb2ed049 100644 --- a/scrapy/extensions/closespider.py +++ b/scrapy/extensions/closespider.py @@ -54,9 +54,9 @@ class CloseSpider(object): self.crawler.engine.close_spider(spider, 'closespider_pagecount') def spider_opened(self, spider): - self.task = reactor.callLater(self.close_on['timeout'], \ - self.crawler.engine.close_spider, spider, \ - reason='closespider_timeout') + self.task = reactor.callLater(self.close_on['timeout'], + self.crawler.engine.close_spider, spider, + reason='closespider_timeout') def item_scraped(self, item, spider): self.counter['itemcount'] += 1 diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e2492d506..11a63ced2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,7 +10,7 @@ import logging import posixpath from tempfile import NamedTemporaryFile from datetime import datetime -from six.moves.urllib.parse import urlparse, unquote +from urllib.parse import urlparse, unquote from ftplib import FTP from zope.interface import Interface, implementer diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index f3fabf710..91850683f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,23 +1,19 @@ -from __future__ import print_function - import gzip import logging import os +import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time -from warnings import warn from weakref import WeakKeyDictionary -from six.moves import cPickle as pickle from w3lib.http import headers_raw_to_dict, headers_dict_to_raw -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Headers, Response from scrapy.responsetypes import responsetypes from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path -from scrapy.utils.python import to_bytes, to_unicode, garbage_collect +from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.request import request_fingerprint diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py index 263d8ce4c..892aa8a86 100644 --- a/scrapy/extensions/memdebug.py +++ b/scrapy/extensions/memdebug.py @@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst """ import gc -import six from scrapy import signals from scrapy.exceptions import NotConfigured @@ -28,7 +27,7 @@ class MemoryDebugger(object): def spider_closed(self, spider, reason): gc.collect() self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider) - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if not wdict: continue self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider) diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2220cbd8f..2c8e46914 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,10 +1,11 @@ import os -from six.moves import cPickle as pickle +import pickle from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir + class SpiderState(object): """Store and load spider state during a scraping job""" diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 60a14c6f8..0903fd4f8 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,7 +1,6 @@ import time -from six.moves.http_cookiejar import ( - CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE -) +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE + from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index f3b46b994..dcaaeddfa 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -1,4 +1,3 @@ -import six from w3lib.http import headers_dict_to_raw from scrapy.utils.datatypes import CaselessDict from scrapy.utils.python import to_unicode @@ -19,7 +18,7 @@ class Headers(CaselessDict): """Normalize values to bytes""" if value is None: value = [] - elif isinstance(value, (six.text_type, bytes)): + elif isinstance(value, (str, bytes)): value = [value] elif not hasattr(value, '__iter__'): value = [value] @@ -29,10 +28,10 @@ class Headers(CaselessDict): def _tobytes(self, x): if isinstance(x, bytes): return x - elif isinstance(x, six.text_type): + elif isinstance(x, str): return x.encode(self.encoding) elif isinstance(x, int): - return six.text_type(x).encode(self.encoding) + return str(x).encode(self.encoding) else: raise TypeError('Unsupported value type: {}'.format(type(x))) @@ -68,9 +67,6 @@ class Headers(CaselessDict): self[key] = lst def items(self): - return list(self.iteritems()) - - def iteritems(self): return ((k, self.getlist(k)) for k in self.keys()) def values(self): diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index d09eaf849..b5c8e1a9a 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,7 +4,6 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ -import six from w3lib.url import safe_url_string from scrapy.http.headers import Headers @@ -60,13 +59,13 @@ class Request(object_ref): return self._url def _set_url(self, url): - if not isinstance(url, six.string_types): + if not isinstance(url, str): raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) - if ':' not in self._url: + if ('://' not in self._url) and (not self._url.startswith('data:')): raise ValueError('Missing scheme in request url: %s' % self._url) url = property(_get_url, obsolete_setter(_set_url, 'url')) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6feede07..af02c8484 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin, urlencode +from urllib.parse import urljoin, urlencode import lxml.html from parsel.selector import create_root_node @@ -208,7 +207,7 @@ def _get_clickable(clickdata, form): # We didn't find it, so now we build an XPath expression out of the other # arguments, because they can be used as such xpath = u'.//*' + \ - u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata)) + u''.join(u'[@%s="%s"]' % c for c in clickdata.items()) el = form.xpath(xpath) if len(el) == 1: return (el[0].get('name'), el[0].get('value') or '') diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bd09f7534..811d3ad6b 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from six.moves import xmlrpc_client as xmlrpclib +import xmlrpc.client as xmlrpclib from scrapy.http.request import Request from scrapy.utils.python import get_func_args diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a81404afb..64e9c6c20 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from scrapy.http.request import Request from scrapy.http.headers import Headers diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index bd3559fbb..7eed052c2 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class HtmlResponse(TextResponse): pass diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 37f450e54..1079fd6e8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import parsel from w3lib.encoding import html_to_unicode, resolve_encoding, \ @@ -31,14 +30,14 @@ class TextResponse(Response): super(TextResponse, self).__init__(*args, **kwargs) def _set_url(self, url): - if isinstance(url, six.text_type): + if isinstance(url, str): self._url = to_unicode(url, self.encoding) else: super(TextResponse, self)._set_url(url) def _set_body(self, body): self._body = b'' # used by encoding detection - if isinstance(body, six.text_type): + if isinstance(body, str): if self._encoding is None: raise TypeError('Cannot convert unicode body - %s has no encoding' % type(self).__name__) @@ -158,7 +157,7 @@ class TextResponse(Response): def _url_from_selector(sel): # type: (parsel.Selector) -> str - if isinstance(sel.root, six.string_types): + if isinstance(sel.root, str): # e.g. ::attr(href) result return strip_html5_whitespace(sel.root) if not hasattr(sel.root, 'tag'): diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 1df33fee5..abf474a2f 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -7,5 +7,6 @@ See documentation in docs/topics/request-response.rst from scrapy.http.response.text import TextResponse + class XmlResponse(TextResponse): pass diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py index d48babc3c..1896ec31e 100644 --- a/scrapy/interfaces.py +++ b/scrapy/interfaces.py @@ -1,5 +1,6 @@ from zope.interface import Interface + class ISpiderLoader(Interface): def from_settings(settings): diff --git a/scrapy/item.py b/scrapy/item.py index 32f9b2ebb..1d39b48b2 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -10,8 +10,6 @@ from copy import deepcopy from pprint import pformat from warnings import warn -import six - from scrapy.utils.deprecate import ScrapyDeprecationWarning from scrapy.utils.trackref import object_ref @@ -78,7 +76,7 @@ class DictItem(MutableMapping, BaseItem): def __init__(self, *args, **kwargs): self._values = {} if args or kwargs: # avoid creating dict for most common case - for k, v in six.iteritems(dict(*args, **kwargs)): + for k, v in dict(*args, **kwargs).items(): self[k] = v def __getitem__(self, key): @@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem): return deepcopy(self) -@six.add_metaclass(ItemMeta) -class Item(DictItem): +class Item(DictItem, metaclass=ItemMeta): pass diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index e4c62f87b..bc65f41cc 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,8 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse from parsel.csstranslator import HTMLTranslator from w3lib.url import canonicalize_url @@ -44,8 +44,7 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any(r.search(url) for r in regexs) -_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', \ - 'file', 'ftp'} +_is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file', 'ftp'} class FilteringLinkExtractor(object): @@ -118,4 +117,4 @@ class FilteringLinkExtractor(object): # Top-level imports -from .lxmlhtml import LxmlLinkExtractor as LinkExtractor +from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401 diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 27978a8a1..0425d4340 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -2,9 +2,8 @@ HTMLParser-based link extractor """ import warnings -import six -from six.moves.html_parser import HTMLParser -from six.moves.urllib.parse import urljoin +from html.parser import HTMLParser +from urllib.parse import urljoin from w3lib.url import safe_url_string from w3lib.html import strip_html5_whitespace @@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser): ret = [] base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 890c019c8..cb55e805a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,8 +1,7 @@ """ Link extractor based on lxml.html """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import lxml.etree as etree from w3lib.html import strip_html5_whitespace @@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()") def _nons(tag): - if isinstance(tag, six.string_types): + if isinstance(tag, str): if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: return tag.split('}')[-1] return tag diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index e689b4727..3f2557248 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,10 +1,11 @@ import re -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link -from .sgml import SgmlLinkExtractor +from scrapy.linkextractors.sgml import SgmlLinkExtractor + linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 8940a4d77..2ba6bca45 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -1,9 +1,8 @@ """ SGMLParser-based Link extractors """ -import six -from six.moves.urllib.parse import urljoin import warnings +from urllib.parse import urljoin from sgmllib import SGMLParser from w3lib.url import safe_url_string, canonicalize_url @@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser): if base_url is None: base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - if isinstance(link.url, six.text_type): + if isinstance(link.url, str): link.url = link.url.encode(response_encoding) try: link.url = urljoin(base_url, link.url) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 60fd6d222..7cf67e29e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -4,8 +4,7 @@ Item Loader See documentation in docs/topics/loaders.rst """ from collections import defaultdict - -import six +from contextlib import suppress from scrapy.item import Item from scrapy.loader.common import wrap_loader_context @@ -15,6 +14,17 @@ from scrapy.utils.misc import arg_to_iter, extract_regex from scrapy.utils.python import flatten +def unbound_method(method): + """ + Allow to use single-argument functions as input or output processors + (no need to define an unused first 'self' argument) + """ + with suppress(AttributeError): + if '.' not in method.__qualname__: + return method.__func__ + return method + + class ItemLoader(object): default_item_class = Item @@ -72,7 +82,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k, v in six.iteritems(value): + for k, v in value.items(): self._add_value(k, v) else: self._add_value(field_name, value) @@ -82,7 +92,7 @@ class ItemLoader(object): if value is None: return if not field_name: - for k, v in six.iteritems(value): + for k, v in value.items(): self._replace_value(k, v) else: self._replace_value(field_name, value) @@ -142,14 +152,14 @@ class ItemLoader(object): if not proc: proc = self._get_item_field_attr(field_name, 'input_processor', self.default_input_processor) - return proc + return unbound_method(proc) def get_output_processor(self, field_name): proc = getattr(self, '%s_out' % field_name, None) if not proc: proc = self._get_item_field_attr(field_name, 'output_processor', self.default_output_processor) - return proc + return unbound_method(proc) def _process_input_value(self, field_name, value): proc = self.get_input_processor(field_name) diff --git a/scrapy/loader/common.py b/scrapy/loader/common.py index 916524947..42f8de636 100644 --- a/scrapy/loader/common.py +++ b/scrapy/loader/common.py @@ -3,6 +3,7 @@ from functools import partial from scrapy.utils.python import get_func_args + def wrap_loader_context(function, context): """Wrap functions that receive loader_context to contain the context "pre-loaded" and expose a interface that receives only one argument diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 3c61ed7e0..5189d7cfa 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -8,6 +8,7 @@ from scrapy.utils.request import referer_str SCRAPEDMSG = u"Scraped from %(src)s" + os.linesep + "%(item)s" DROPPEDMSG = u"Dropped: %(exception)s" + os.linesep + "%(item)s" CRAWLEDMSG = u"Crawled (%(status)s) %(request)s%(request_flags)s (referer: %(referer)s)%(response_flags)s" +ERRORMSG = u"'Error processing %(item)s'" class LogFormatter(object): @@ -92,6 +93,16 @@ class LogFormatter(object): } } + def error(self, item, exception, response, spider): + """Logs a message when an item causes an error while it is passing through the item pipeline.""" + return { + 'level': logging.ERROR, + 'msg': ERRORMSG, + 'args': { + 'item': item, + } + } + @classmethod def from_crawler(cls, crawler): return cls() diff --git a/scrapy/mail.py b/scrapy/mail.py index d24de2212..9655b8114 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,21 +3,21 @@ Mail sending helpers See documentation in docs/topics/email.rst """ -from io import BytesIO import logging - -from email.utils import COMMASPACE, formatdate -from six.moves.email_mime_multipart import MIMEMultipart -from six.moves.email_mime_text import MIMEText -from six.moves.email_mime_base import MIMEBase -from email.mime.nonmultipart import MIMENonMultipart from email import encoders as Encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate +from io import BytesIO from twisted.internet import defer, reactor, ssl from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes + logger = logging.getLogger(__name__) @@ -73,8 +73,7 @@ class MailSender(object): part = MIMEBase(*mimetype.split('/')) part.set_payload(f.read()) Encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="%s"' \ - % attach_name) + part.add_header('Content-Disposition', 'attachment', filename=attach_name) msg.attach(part) else: msg.set_payload(body) diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 1cfd8a782..53fa435bb 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -65,8 +65,8 @@ class MiddlewareManager(object): return process_chain(self.methods[methodname], obj, *args) def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args): - return process_chain_both(self.methods[cb_methodname], \ - self.methods[eb_methodname], obj, *args) + return process_chain_both(self.methods[cb_methodname], + self.methods[eb_methodname], obj, *args) def open_spider(self, spider): return self._process_parallel('open_spider', spider) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 2ef8786d0..aa1bfb77f 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -7,6 +7,7 @@ See documentation in docs/item-pipeline.rst from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list + class ItemPipelineManager(MiddlewareManager): component_name = 'item pipeline' diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8d74c5011..6d55c8980 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib -from io import BytesIO +import logging import mimetypes import os -import os.path import time -import logging -from email.utils import parsedate_tz, mktime_tz -from six.moves.urllib.parse import urlparse from collections import defaultdict -import six +from email.utils import parsedate_tz, mktime_tz +from io import BytesIO +from urllib.parse import urlparse from twisted.internet import defer, threads @@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str from scrapy.utils.boto import is_botocore from scrapy.utils.datatypes import CaselessDict + logger = logging.getLogger(__name__) @@ -153,14 +152,14 @@ class S3FilesStore(object): Bucket=self.bucket, Key=key_name, Body=buf, - Metadata={k: str(v) for k, v in six.iteritems(meta or {})}, + Metadata={k: str(v) for k, v in (meta or {}).items()}, ACL=self.POLICY, **extra) else: b = self._get_boto_bucket() k = b.new_key(key_name) if meta: - for metakey, metavalue in six.iteritems(meta): + for metakey, metavalue in meta.items(): k.set_metadata(metakey, str(metavalue)) h = self.HEADERS.copy() if headers: @@ -201,7 +200,7 @@ class S3FilesStore(object): 'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation', }) extra = {} - for key, value in six.iteritems(headers): + for key, value in headers.items(): try: kwarg = mapping[key] except KeyError: @@ -249,7 +248,7 @@ class GCSFilesStore(object): def persist_file(self, path, buf, info, meta=None, headers=None): blob = self.bucket.blob(self.prefix + path) blob.cache_control = self.CACHE_CONTROL - blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})} + blob.metadata = {k: str(v) for k, v in (meta or {}).items()} return threads.deferToThread( blob.upload_from_string, data=buf.getvalue(), diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e77cef4ff..e9c6b759c 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst import functools import hashlib from io import BytesIO -import six from PIL import Image @@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline): image, buf = self.convert_image(orig_image) yield path, image, buf - for thumb_id, size in six.iteritems(self.thumbs): + for thumb_id, size in self.thumbs.items(): thumb_path = self.thumb_path(request, thumb_id, response=response, info=info) thumb_image, thumb_buf = self.convert_image(image, size) yield thumb_path, thumb_image, thumb_buf diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 95dca9a3f..c174addf9 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -1,5 +1,3 @@ -from __future__ import print_function - import functools import logging from collections import defaultdict diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 0aaced7e4..4df949015 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -7,6 +7,7 @@ from scrapy.utils.datatypes import LocalCache dnscache = LocalCache(10000) + class CachingThreadedResolver(ThreadedResolver): def __init__(self, reactor, cache_size, timeout): super(CachingThreadedResolver, self).__init__(reactor) diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index de62276c8..91d309147 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -2,11 +2,9 @@ This module implements a class which returns the appropriate Response class based on different criteria. """ -from __future__ import absolute_import from mimetypes import MimeTypes from pkgutil import get_data from io import StringIO -import six from scrapy.http import Response from scrapy.utils.misc import load_object @@ -37,7 +35,7 @@ class ResponseTypes(object): self.mimetypes = MimeTypes() mimedata = get_data('scrapy', 'mime.types').decode('utf8') self.mimetypes.readfp(StringIO(mimedata)) - for mimetype, cls in six.iteritems(self.CLASSES): + for mimetype, cls in self.CLASSES.items(): self.classes[mimetype] = load_object(cls) def from_mimetype(self, mimetype): diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 95a8c09b8..0a9af3a62 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -1,12 +1,13 @@ import sys import logging from abc import ABCMeta, abstractmethod -from six import with_metaclass from scrapy.utils.python import to_unicode + logger = logging.getLogger(__name__) + def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): try: if to_native_str_type: @@ -23,7 +24,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False): robotstxt_body = '' return robotstxt_body -class RobotParser(with_metaclass(ABCMeta)): + +class RobotParser(metaclass=ABCMeta): @classmethod @abstractmethod def from_crawler(cls, crawler, robotstxt_body): @@ -53,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)): class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): - from six.moves.urllib_robotparser import RobotFileParser + from urllib.robotparser import RobotFileParser self.spider = spider robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp = RobotFileParser() diff --git a/scrapy/selector/__init__.py b/scrapy/selector/__init__.py index 90e96ee92..a9240c1f6 100644 --- a/scrapy/selector/__init__.py +++ b/scrapy/selector/__init__.py @@ -1,4 +1,4 @@ """ Selectors """ -from scrapy.selector.unified import * +from scrapy.selector.unified import * # noqa: F401 diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py index 62fda40b0..a08955dc9 100644 --- a/scrapy/selector/unified.py +++ b/scrapy/selector/unified.py @@ -2,12 +2,10 @@ XPath selectors based on lxml """ -import warnings from parsel import Selector as _ParselSelector from scrapy.utils.trackref import object_ref from scrapy.utils.python import to_bytes from scrapy.http import HtmlResponse, XmlResponse -from scrapy.utils.decorators import deprecated __all__ = ['Selector', 'SelectorList'] diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index c871e86e0..b6133619c 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -1,4 +1,3 @@ -import six import json import copy from collections.abc import MutableMapping @@ -23,7 +22,7 @@ def get_settings_priority(priority): :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its numerical value, or directly returns a given numerical priority. """ - if isinstance(priority, six.string_types): + if isinstance(priority, str): return SETTINGS_PRIORITIES[priority] else: return priority @@ -173,7 +172,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or []) - if isinstance(value, six.string_types): + if isinstance(value, str): value = value.split(',') return list(value) @@ -194,7 +193,7 @@ class BaseSettings(MutableMapping): :type default: any """ value = self.get(name, default or {}) - if isinstance(value, six.string_types): + if isinstance(value, str): value = json.loads(value) return dict(value) @@ -284,7 +283,7 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(module, six.string_types): + if isinstance(module, str): module = import_module(module) for key in dir(module): if key.isupper(): @@ -313,14 +312,14 @@ class BaseSettings(MutableMapping): :type priority: string or int """ self._assert_mutability() - if isinstance(values, six.string_types): + if isinstance(values, str): values = json.loads(values) if values is not None: if isinstance(values, BaseSettings): - for name, value in six.iteritems(values): + for name, value in values.items(): self.set(name, value, values.getpriority(name)) else: - for name, value in six.iteritems(values): + for name, value in values.items(): self.set(name, value, priority) def delete(self, name, priority='project'): @@ -377,7 +376,7 @@ class BaseSettings(MutableMapping): def _to_dict(self): return {k: (v._to_dict() if isinstance(v, BaseSettings) else v) - for k, v in six.iteritems(self)} + for k, v in self.items()} def copy_to_dict(self): """ @@ -445,7 +444,7 @@ class Settings(BaseSettings): self.setmodule(default_settings, 'default') # Promote default dictionaries to BaseSettings instances for per-key # priorities - for name, val in six.iteritems(self): + for name, val in self.items(): if isinstance(val, dict): self.set(name, BaseSettings(val, 'default'), 'default') self.update(values, priority) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5c9678c01..1e163e1fb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT' -DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, - # also allowing negotiation +# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: +DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} diff --git a/scrapy/shell.py b/scrapy/shell.py index 80b625633..a649d555f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -3,8 +3,6 @@ See documentation in docs/topics/shell.rst """ -from __future__ import print_function - import os import signal import warnings diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 296d27ed8..481d97e9a 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import from pydispatch import dispatcher from scrapy.utils import signal as _signal @@ -46,16 +45,14 @@ class SignalManager(object): def send_catch_log_deferred(self, signal, **kwargs): """ - Like :meth:`send_catch_log` but supports returning `deferreds`_ from - signal handlers. + Like :meth:`send_catch_log` but supports returning + :class:`~twisted.internet.defer.Deferred` objects from signal handlers. Returns a Deferred that gets fired once all signal handlers deferreds were fired. Send a signal, catch exceptions and log them. The keyword arguments are passed to the signal handlers (connected through the :meth:`connect` method). - - .. _deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html """ kwargs.setdefault('sender', self.sender) return _signal.send_catch_log_deferred(signal, **kwargs) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 7478faa78..3beca4060 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import from collections import defaultdict import traceback import warnings diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c76e4d5a2..dce2b3598 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,8 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse import warnings +from urllib.parse import urlparse from w3lib.url import safe_url_string diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 94095bc27..9429f6cb2 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -10,7 +10,6 @@ from scrapy import signals from scrapy.http import Request from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import method_is_overridden @@ -58,6 +57,11 @@ class Spider(object_ref): def start_requests(self): cls = self.__class__ + if not self.start_urls and hasattr(self, 'start_url'): + raise AttributeError( + "Crawling could not start: 'start_urls' not found " + "or empty (but found 'start_url' attribute instead, " + "did you miss an 's'?)") if method_is_overridden(cls, Spider, 'make_requests_from_url'): warnings.warn( "Spider.make_requests_from_url method is deprecated; it " @@ -100,6 +104,6 @@ class Spider(object_ref): # Top-level imports -from scrapy.spiders.crawl import CrawlSpider, Rule -from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider -from scrapy.spiders.sitemap import SitemapSpider +from scrapy.spiders.crawl import CrawlSpider, Rule # noqa: F401 +from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider # noqa: F401 +from scrapy.spiders.sitemap import SitemapSpider # noqa: F401 diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 03000ce54..a5eb1a518 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst import copy import warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.linkextractors import LinkExtractor @@ -25,7 +23,7 @@ def _identity(request, response): def _get_method(method, spider): if callable(method): return method - elif isinstance(method, six.string_types): + elif isinstance(method, str): return getattr(spider, method, None) diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 197812a26..c566f0236 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -100,8 +100,8 @@ class CSVFeedSpider(Spider): and the file's headers. """ - delimiter = None # When this is None, python's csv module's default delimiter is used - quotechar = None # When this is None, python's csv module's default quotechar is used + delimiter = None # When this is None, python's csv module's default delimiter is used + quotechar = None # When this is None, python's csv module's default quotechar is used headers = None def process_results(self, response, results): diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 534c45c70..d368c7108 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -1,6 +1,5 @@ import re import logging -import six from scrapy.spiders import Spider from scrapy.http import Request, XmlResponse @@ -22,7 +21,7 @@ class SitemapSpider(Spider): super(SitemapSpider, self).__init__(*a, **kw) self._cbs = [] for r, c in self.sitemap_rules: - if isinstance(c, six.string_types): + if isinstance(c, str): c = getattr(self, c) self._cbs.append((regex(r), c)) self._follow = [regex(x) for x in self.sitemap_follow] @@ -86,7 +85,7 @@ class SitemapSpider(Spider): def regex(x): - if isinstance(x, six.string_types): + if isinstance(x, str): return re.compile(x) return x diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 30cc926e5..d5d3be67e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -from six.moves import cPickle as pickle +import pickle from queuelib import queue diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 5bbda6e27..cdbe21942 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,5 +1,6 @@ import random -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode + from twisted.web.server import Site from twisted.web.resource import Resource from twisted.internet import reactor diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index c8fc911bb..12321caa5 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -1,13 +1,11 @@ """Boto/botocore helpers""" -from __future__ import absolute_import - from scrapy.exceptions import NotConfigured def is_botocore(): try: - import botocore + import botocore # noqa: F401 return True except ImportError: raise NotConfigured('missing botocore library') diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 561bb72fc..23306ca28 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -1,11 +1,9 @@ -from configparser import ConfigParser import os import sys import numbers +from configparser import ConfigParser from operator import itemgetter -import six - from scrapy.settings import BaseSettings from scrapy.utils.deprecate import update_classpath from scrapy.utils.python import without_none_values @@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): def _map_keys(compdict): if isinstance(compdict, BaseSettings): compbs = BaseSettings() - for k, v in six.iteritems(compdict): + for k, v in compdict.items(): prio = compdict.getpriority(k) if compbs.getpriority(convert(k)) == prio: raise ValueError('Some paths in {!r} convert to the same ' @@ -33,13 +31,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath): return compbs else: _check_components(compdict) - return {convert(k): v for k, v in six.iteritems(compdict)} + return {convert(k): v for k, v in compdict.items()} def _validate_values(compdict): """Fail if a value in the components dict is not a real number or None.""" - for name, value in six.iteritems(compdict): + for name, value in compdict.items(): if value is not None and not isinstance(value, numbers.Real): - raise ValueError('Invalid value {} for component {}, please provide ' \ + raise ValueError('Invalid value {} for component {}, please provide ' 'a real number or None instead'.format(value, name)) # BEGIN Backward compatibility for old (base, custom) call signature @@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath): _validate_values(compdict) compdict = without_none_values(_map_keys(compdict)) - return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))] + return [k for k, v in sorted(compdict.items(), key=itemgetter(1))] def arglist_to_dict(arglist): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 2e9981556..7eb40f0ce 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,6 +1,7 @@ from functools import wraps from collections import OrderedDict + def _embed_ipython_shell(namespace={}, banner=''): """Start an IPython Shell""" try: @@ -23,6 +24,7 @@ def _embed_ipython_shell(namespace={}, banner=''): shell() return wrapper + def _embed_bpython_shell(namespace={}, banner=''): """Start a bpython shell""" import bpython @@ -31,6 +33,7 @@ def _embed_bpython_shell(namespace={}, banner=''): bpython.embed(locals_=namespace, banner=banner) return wrapper + def _embed_ptpython_shell(namespace={}, banner=''): """Start a ptpython shell""" import ptpython.repl @@ -40,21 +43,23 @@ def _embed_ptpython_shell(namespace={}, banner=''): ptpython.repl.embed(locals=namespace) return wrapper + def _embed_standard_shell(namespace={}, banner=''): """Start a standard python shell""" import code - try: # readline module is only available on unix systems + try: # readline module is only available on unix systems import readline except ImportError: pass else: - import rlcompleter + import rlcompleter # noqa: F401 readline.parse_and_bind("tab:complete") @wraps(_embed_standard_shell) def wrapper(namespace=namespace, banner=''): code.interact(banner=banner, local=namespace) return wrapper + DEFAULT_PYTHON_SHELLS = OrderedDict([ ('ptpython', _embed_ptpython_shell), ('ipython', _embed_ipython_shell), @@ -62,13 +67,14 @@ DEFAULT_PYTHON_SHELLS = OrderedDict([ ('python', _embed_standard_shell), ]) + def get_shell_embed_func(shells=None, known_shells=None): """Return the first acceptable shell-embed function from a given list of shell names. """ - if shells is None: # list, preference order of shells + if shells is None: # list, preference order of shells shells = DEFAULT_PYTHON_SHELLS.keys() - if known_shells is None: # available embeddable shells + if known_shells is None: # available embeddable shells known_shells = DEFAULT_PYTHON_SHELLS.copy() for shell in shells: if shell in known_shells: @@ -79,6 +85,7 @@ def get_shell_embed_func(shells=None, known_shells=None): except ImportError: continue + def start_python_console(namespace=None, banner='', shells=None): """Start Python console bound to the given namespace. Readline support and tab completion will be used on Unix, if available. @@ -90,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None): shell = get_shell_embed_func(shells) if shell is not None: shell(namespace=namespace, banner=banner) - except SystemExit: # raised when using exit() in python code.interact + except SystemExit: # raised when using exit() in python code.interact pass diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index b3fd0a497..16639356e 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,10 +1,9 @@ import argparse import warnings from shlex import split +from http.cookies import SimpleCookie +from urllib.parse import urlparse -from six.moves.http_cookies import SimpleCookie -from six.moves.urllib.parse import urlparse -from six import string_types, iteritems from w3lib.http import basic_auth_header @@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): name = name.strip() val = val.strip() if name.title() == 'Cookie': - for name, morsel in iteritems(SimpleCookie(val)): + for name, morsel in SimpleCookie(val).items(): cookies[name] = morsel.value else: headers.append((name, val)) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 39d389fa6..ffd1537c3 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -5,12 +5,10 @@ Python Standard Library. This module must not depend on any module outside the Standard Library. """ -import copy import collections -from collections.abc import Mapping +import copy import warnings - -import six +from collections.abc import Mapping from scrapy.exceptions import ScrapyDeprecationWarning @@ -151,7 +149,7 @@ class MultiValueDict(dict): self.setlistdefault(key, []).append(value) except TypeError: raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary") - for key, value in six.iteritems(kwargs): + for key, value in kwargs.items(): self.setlistdefault(key, []).append(value) @@ -238,65 +236,6 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class MergeDict(object): - """ - A simple class for creating new "virtual" dictionaries that actually look - up values in more than one dictionary, passed in the ``__init__`` method. - - If a key appears in more than one of the given dictionaries, only the - first occurrence will be used. - """ - def __init__(self, *dicts): - warnings.warn( - "scrapy.utils.datatypes.MergeDict is deprecated in favor " - "of collections.ChainMap (introduced in Python 3.3)", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - self.dicts = dicts - - def __getitem__(self, key): - for dict_ in self.dicts: - try: - return dict_[key] - except KeyError: - pass - raise KeyError - - def __copy__(self): - return self.__class__(*self.dicts) - - def get(self, key, default=None): - try: - return self[key] - except KeyError: - return default - - def getlist(self, key): - for dict_ in self.dicts: - if key in dict_.keys(): - return dict_.getlist(key) - return [] - - def items(self): - item_list = [] - for dict_ in self.dicts: - item_list.extend(dict_.items()) - return item_list - - def has_key(self, key): - for dict_ in self.dicts: - if key in dict_: - return True - return False - - __contains__ = has_key - - def copy(self): - """Returns a copy of this object.""" - return self.__copy__() - - class LocalCache(collections.OrderedDict): """Dictionary with a finite number of keys. diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 38bee1a6c..2e2c7adc1 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -34,6 +34,7 @@ def defers(func): return defer.maybeDeferred(func, *a, **kw) return wrapped + def inthread(func): """Decorator to call a function in a thread and return a deferred with the result diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 69d621830..c5916c21c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -7,6 +7,7 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest + def defer_fail(_failure): """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -18,6 +19,7 @@ def defer_fail(_failure): reactor.callLater(0.1, d.errback, _failure) return d + def defer_succeed(result): """Same as twisted.internet.defer.succeed but delay calling callback until next reactor loop @@ -29,6 +31,7 @@ def defer_succeed(result): reactor.callLater(0.1, d.callback, result) return d + def defer_result(result): if isinstance(result, defer.Deferred): return result @@ -37,6 +40,7 @@ def defer_result(result): else: return defer_succeed(result) + def mustbe_deferred(f, *args, **kw): """Same as twisted.internet.defer.maybeDeferred, but delay calling callback/errback to next reactor loop @@ -53,6 +57,7 @@ def mustbe_deferred(f, *args, **kw): else: return defer_result(result) + def parallel(iterable, count, callable, *args, **named): """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. @@ -63,6 +68,7 @@ def parallel(iterable, count, callable, *args, **named): work = (callable(elem, *args, **named) for elem in iterable) return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) + def process_chain(callbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" d = defer.Deferred() @@ -71,6 +77,7 @@ def process_chain(callbacks, input, *a, **kw): d.callback(input) return d + def process_chain_both(callbacks, errbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks and errbacks""" d = defer.Deferred() @@ -83,6 +90,7 @@ def process_chain_both(callbacks, errbacks, input, *a, **kw): d.callback(input) return d + def process_parallel(callbacks, input, *a, **kw): """Return a Deferred with the output of all successful calls to the given callbacks @@ -92,6 +100,7 @@ def process_parallel(callbacks, input, *a, **kw): d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure) return d + def iter_errback(iterable, errback, *a, **kw): """Wraps an iterable calling an errback if an error is caught while iterating it. diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py index f6a6c4645..9735220ef 100644 --- a/scrapy/utils/display.py +++ b/scrapy/utils/display.py @@ -2,10 +2,10 @@ pprint and pformat wrappers with colorization support """ -from __future__ import print_function import sys from pprint import pformat as pformat_ + def _colorize(text, colorize=True): if not colorize or not sys.stdout.isatty(): return text @@ -17,8 +17,10 @@ def _colorize(text, colorize=True): except ImportError: return text + def pformat(obj, *args, **kwargs): return _colorize(pformat_(obj), kwargs.pop('colorize', True)) + def pprint(obj, *args, **kwargs): print(pformat(obj, *args, **kwargs)) diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 11dd36d91..267c7ecd1 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -1,7 +1,8 @@ """Some debugging functions for working with the Scrapy engine""" -from __future__ import print_function -from time import time # used in global tests code +# used in global tests code +from time import time # noqa: F401 + def get_engine_status(engine): """Return a report of the current engine status""" @@ -32,6 +33,7 @@ def get_engine_status(engine): return checks + def format_engine_status(engine=None): checks = get_engine_status(engine) s = "Execution engine status\n\n" @@ -41,5 +43,6 @@ def format_engine_status(engine=None): return s + def print_engine_status(engine): print(format_engine_status(engine)) diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py index 9eca6a4da..91d2439a9 100644 --- a/scrapy/utils/ftp.py +++ b/scrapy/utils/ftp.py @@ -1,6 +1,7 @@ from ftplib import error_perm from posixpath import dirname + def ftp_makedirs_cwd(ftp, path, first_call=True): """Set the current directory of the FTP connection given in the ``ftp`` argument (as a ftplib.FTP object), creating all parent directories if they diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index f41e62fe3..9672e28da 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -45,6 +45,7 @@ def gunzip(data): _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search + @deprecated def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index ad49ef3e9..bab262393 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -8,7 +8,7 @@ import warnings from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.decorators import deprecated -from w3lib.http import * +from w3lib.http import * # noqa: F401 warnings.warn("Module `scrapy.utils.http` is deprecated, " diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index b4c929b0e..c8d4391b1 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,10 +1,12 @@ """Helper functions for scrapy.http objects (Request, Response)""" import weakref +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse _urlparse_cache = weakref.WeakKeyDictionary() + + def urlparse_cached(request_or_response): """Return urlparse.urlparse caching the result, where the argument can be a Request or Response object diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9693ba768..3c0cb68c3 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,13 +1,13 @@ -import re import csv -from io import StringIO import logging -import six +import re +from io import StringIO from scrapy.http import TextResponse, Response from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode + logger = logging.getLogger(__name__) @@ -60,7 +60,7 @@ class _StreamReader(object): self._text, self.encoding = obj.body, obj.encoding else: self._text, self.encoding = obj, 'utf-8' - self._is_unicode = isinstance(self._text, six.text_type) + self._is_unicode = isinstance(self._text, str) def read(self, n=65535): self.read = self._read_unicode if self._is_unicode else self._read_string @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - expected_types = (Response, six.text_type, six.binary_type) + expected_types = (Response, str, bytes) assert isinstance(obj, expected_types), \ "obj must be %s, not %s" % ( " or ".join(t.__name__ for t in expected_types), @@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True): return obj.text else: return obj.body.decode('utf-8') - elif isinstance(obj, six.text_type): + elif isinstance(obj, str): return obj if unicode else obj.encode('utf-8') else: return obj.decode('utf-8') if unicode else obj diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py index 389fde73a..4f1e601fc 100644 --- a/scrapy/utils/job.py +++ b/scrapy/utils/job.py @@ -1,5 +1,6 @@ import os + def job_dir(settings): path = settings['JOBDIR'] if path and not os.path.exists(path): diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index a18f308a3..2455fcc16 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -6,7 +6,7 @@ For new code, always import from w3lib.html instead of this module import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.html import * +from w3lib.html import * # noqa: F401 warnings.warn("Module `scrapy.utils.markup` is deprecated. " diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b74f34451..9955fb1e7 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -6,14 +6,13 @@ from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules -import six from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem -_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes +_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes def arg_to_iter(arg): @@ -83,7 +82,7 @@ def extract_regex(regex, text, encoding='utf-8'): * if the regex doesn't contain any group the entire regex matching is returned """ - if isinstance(regex, six.string_types): + if isinstance(regex, str): regex = re.compile(regex, re.UNICODE) try: @@ -92,7 +91,7 @@ def extract_regex(regex, text, encoding='utf-8'): strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) - if isinstance(text, six.text_type): + if isinstance(text, str): return [replace_entities(s, keep=['lt', 'amp']) for s in strings] else: return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp']) diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py index c2d8afd07..e81f63152 100644 --- a/scrapy/utils/multipart.py +++ b/scrapy/utils/multipart.py @@ -6,7 +6,7 @@ For new code, always import from w3lib.form instead of this module import warnings from scrapy.exceptions import ScrapyDeprecationWarning -from w3lib.form import * +from w3lib.form import * # noqa: F401 warnings.warn("Module `scrapy.utils.multipart` is deprecated. " diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index f87d5a803..7a7aec9be 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -1,5 +1,3 @@ - -from __future__ import absolute_import import signal from twisted.internet import reactor diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 1cbda141a..f28c2eaa1 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,5 @@ import os -from six.moves import cPickle as pickle +import pickle import warnings from importlib import import_module @@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning + ENVVAR = 'SCRAPY_SETTINGS_MODULE' DATADIR_CFG_SECTION = 'datadir' diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 663a8ebaa..8d829c5a5 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,7 +7,6 @@ import re import inspect import weakref import errno -import six from functools import partial, wraps from itertools import chain import sys @@ -65,10 +64,10 @@ def is_listlike(x): True >>> is_listlike((x for x in range(3))) True - >>> is_listlike(six.moves.xrange(5)) + >>> is_listlike(range(5)) True """ - return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) + return hasattr(x, "__iter__") and not isinstance(x, (str, bytes)) def unique(list_, key=lambda x: x): @@ -87,9 +86,9 @@ def unique(list_, key=lambda x: x): def to_unicode(text, encoding=None, errors='strict'): """Return the unicode representation of a bytes object ``text``. If ``text`` is already an unicode object, return it as-is.""" - if isinstance(text, six.text_type): + if isinstance(text, str): return text - if not isinstance(text, (bytes, six.text_type)): + if not isinstance(text, (bytes, str)): raise TypeError('to_unicode must receive a bytes or str ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -102,7 +101,7 @@ def to_bytes(text, encoding=None, errors='strict'): is already a bytes object, return it as-is.""" if isinstance(text, bytes): return text - if not isinstance(text, six.string_types): + if not isinstance(text, str): raise TypeError('to_bytes must receive a str or bytes ' 'object, got %s' % type(text).__name__) if encoding is None: @@ -138,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024): yield (text[offset:], offset) yield (text, 0) - if isinstance(pattern, six.string_types): + if isinstance(pattern, str): pattern = re.compile(pattern) for chunk, offset in _chunk_iter(): @@ -162,9 +161,10 @@ def memoizemethod_noargs(method): return new_method -_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} +_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"} _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS} + @deprecated("scrapy.utils.python.binary_is_text") def isbinarytext(text): """ This function is deprecated. @@ -300,10 +300,10 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True): dict or a list of tuples, like any dict ``__init__`` method supports. """ d = {} - for k, v in six.iteritems(dict(dct_or_tuples)): - k = k.encode(encoding) if isinstance(k, six.text_type) else k + for k, v in dict(dct_or_tuples).items(): + k = k.encode(encoding) if isinstance(k, str) else k if not keys_only: - v = v.encode(encoding) if isinstance(v, six.text_type) else v + v = v.encode(encoding) if isinstance(v, str) else v d[k] = v return d @@ -345,7 +345,7 @@ def without_none_values(iterable): value ``None`` have been removed. """ try: - return {k: v for k, v in six.iteritems(iterable) if v is not None} + return {k: v for k, v in iterable.items() if v is not None} except AttributeError: return type(iterable)((v for v in iterable if v is not None)) @@ -382,9 +382,11 @@ class MutableChain(object): self.data = chain(self.data, *iterables) def __iter__(self): - return self.data.__iter__() + return self def __next__(self): return next(self.data) - next = __next__ + @deprecated("scrapy.utils.python.MutableChain.__next__") + def next(self): + return self.__next__() diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 83186a372..493d26d4c 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -1,12 +1,13 @@ from twisted.internet import reactor, error + def listen_tcp(portrange, host, factory): """Like reactor.listenTCP but tries different ports in a range.""" assert len(portrange) <= 2, "invalid portrange: %s" % portrange - if not hasattr(portrange, '__iter__'): - return reactor.listenTCP(portrange, factory, interface=host) if not portrange: return reactor.listenTCP(0, factory, interface=host) + if not hasattr(portrange, '__iter__'): + return reactor.listenTCP(portrange, factory, interface=host) if len(portrange) == 1: return reactor.listenTCP(portrange[0], factory, interface=host) for x in range(portrange[0], portrange[1]+1): diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 495564ac0..749bbc387 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -1,8 +1,6 @@ """ Helper functions for serializing (and deserializing) requests. """ -import six - from scrapy.http import Request from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object @@ -87,12 +85,12 @@ def _mangle_private_name(obj, func, name): def _find_method(obj, func): if obj: try: - func_self = six.get_method_self(func) + func_self = func.__self__ except AttributeError: # func has no __self__ pass else: if func_self is obj: - name = six.get_method_function(func).__name__ + name = func.__func__.__name__ if _is_private_method(name): return _mangle_private_name(obj, func, name) return name diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 63d0ae772..356753ab5 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -3,19 +3,20 @@ This module provides some useful functions for working with scrapy.http.Request objects """ -from __future__ import print_function import hashlib import weakref -from six.moves.urllib.parse import urlunparse +from urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_unicode - from w3lib.url import canonicalize_url + from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes, to_unicode _fingerprint_cache = weakref.WeakKeyDictionary() + + def request_fingerprint(request, include_headers=None, keep_fragments=False): """ Return the request fingerprint. diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index feab07431..29fdaaf2c 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -13,6 +13,8 @@ from w3lib import html _baseurl_cache = weakref.WeakKeyDictionary() + + def get_base_url(response): """Return the base url of the given response, joined with the response url""" if response not in _baseurl_cache: @@ -23,6 +25,8 @@ def get_base_url(response): _metaref_cache = weakref.WeakKeyDictionary() + + def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 4742b3e13..2f10cf4de 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +from urllib.parse import urljoin + import lxml.etree -from six.moves.urllib.parse import urljoin class Sitemap(object): diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 94b24f67e..4061d1ea3 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,11 +1,10 @@ import logging import inspect -import six - from scrapy.spiders import Spider from scrapy.utils.misc import arg_to_iter + logger = logging.getLogger(__name__) @@ -21,13 +20,14 @@ def iter_spider_classes(module): # singleton in scrapy.spider.spiders from scrapy.spiders import Spider - for obj in six.itervalues(vars(module)): + for obj in vars(module).values(): if inspect.isclass(obj) and \ issubclass(obj, Spider) and \ obj.__module__ == module.__name__ and \ getattr(obj, 'name', None): yield obj + def spidercls_for_request(spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False): """Return a spider class that handles the given Request. diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 615372fc8..96ff4b09b 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -19,6 +19,8 @@ def render_templatefile(path, **kwargs): CAMELCASE_INVALID_CHARS = re.compile(r'[^a-zA-Z\d]') + + def string_camelcase(string): """ Convert a word to its CamelCase version and remove invalid chars diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4b935c51b..307c25352 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -2,7 +2,6 @@ This module contains some assorted functions used in tests """ -from __future__ import absolute_import import os from importlib import import_module @@ -32,6 +31,7 @@ def skip_if_no_boto(): except NotConfigured as e: raise SkipTest(e) + def get_s3_content_and_delete(bucket, path, with_key=False): """ Get content from s3 key, and delete key afterwards. """ @@ -51,6 +51,7 @@ def get_s3_content_and_delete(bucket, path, with_key=False): bucket.delete_key(path) return (content, key) if with_key else content + def get_gcs_content_and_delete(bucket, path): from google.cloud import storage client = storage.Client(project=os.environ.get('GCS_PROJECT_ID')) @@ -61,6 +62,7 @@ def get_gcs_content_and_delete(bucket, path): bucket.delete_blob(path) return content, acl, blob + def get_crawler(spidercls=None, settings_dict=None): """Return an unconfigured Crawler object. If settings_dict is given, it will be used to populate the crawler settings with a project level @@ -72,12 +74,14 @@ def get_crawler(spidercls=None, settings_dict=None): runner = CrawlerRunner(settings_dict) return runner.create_crawler(spidercls or Spider) + def get_pythonpath(): """Return a PYTHONPATH suitable to use in processes so that they find this installation of Scrapy""" scrapy_path = import_module('scrapy').__path__[0] return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '') + def get_testenv(): """Return a OS environment dict suitable to fork processes that need to import this installation of Scrapy, instead of a system installed one. @@ -86,6 +90,7 @@ def get_testenv(): env['PYTHONPATH'] = get_pythonpath() return env + def assert_samelines(testcase, text1, text2, msg=None): """Asserts text1 and text2 have the same lines, ignoring differences in line endings between platforms diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index f268e91ff..0f15cf60a 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import import sys import os diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index e50a989b3..6f5c21624 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,5 +1,4 @@ -from __future__ import print_function -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from twisted.internet import reactor from twisted.web import server, resource, static, util diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index eed14c5a1..4842b95df 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -9,12 +9,10 @@ and no performance penalty at all when disabled (as object_ref becomes just an alias to object in that case). """ -from __future__ import print_function import weakref from time import time from operator import itemgetter from collections import defaultdict -import six NoneType = type(None) @@ -37,13 +35,13 @@ def format_live_refs(ignore=NoneType): """Return a tabular representation of tracked objects""" s = "Live References\n\n" now = time() - for cls, wdict in sorted(six.iteritems(live_refs), + for cls, wdict in sorted(live_refs.items(), key=lambda x: x[0].__name__): if not wdict: continue if issubclass(cls, ignore): continue - oldest = min(six.itervalues(wdict)) + oldest = min(wdict.values()) s += "%-30s %6d oldest: %ds ago\n" % ( cls.__name__, len(wdict), now - oldest ) @@ -57,15 +55,15 @@ def print_live_refs(*a, **kw): def get_oldest(class_name): """Get the oldest object for a specific class name""" - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if cls.__name__ == class_name: if not wdict: break - return min(six.iteritems(wdict), key=itemgetter(1))[0] + return min(wdict.items(), key=itemgetter(1))[0] def iter_all(class_name): """Iterate over all objects of the same class by its class name""" - for cls, wdict in six.iteritems(live_refs): + for cls, wdict in live_refs.items(): if cls.__name__ == class_name: - return six.iterkeys(wdict) + return wdict.keys() diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b3a4be007..c9abb12d5 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,12 +7,12 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) +from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code from w3lib.url import * -from w3lib.url import _safe_chars, _unquotepath +from w3lib.url import _safe_chars, _unquotepath # noqa: F401 from scrapy.utils.python import to_unicode diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py index 48484b303..b0737d3d5 100644 --- a/scrapy/utils/versions.py +++ b/scrapy/utils/versions.py @@ -27,5 +27,5 @@ def scrapy_components_versions(): ("Python", sys.version.replace("\n", "- ")), ("pyOpenSSL", get_openssl_version()), ("cryptography", cryptography.__version__), - ("Platform", platform.platform()), + ("Platform", platform.platform()), ] diff --git a/setup.py b/setup.py index 8f5f14f0d..85d797f88 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ setup( 'pyOpenSSL>=16.2.0', 'queuelib>=1.4.2', 'service_identity>=16.0.0', - 'six>=1.10.0', 'w3lib>=1.17.0', 'zope.interface>=4.1.3', 'protego>=0.1.15', diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py new file mode 100644 index 000000000..5f6f1ae30 --- /dev/null +++ b/tests/CrawlerProcess/simple.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = 'no_request' + + def start_requests(self): + return [] + + +process = CrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/py3-ignores.txt b/tests/ignores.txt similarity index 74% rename from tests/py3-ignores.txt rename to tests/ignores.txt index 313e74ec9..45cf6fb92 100644 --- a/tests/py3-ignores.txt +++ b/tests/ignores.txt @@ -1,6 +1,3 @@ -tests/test_linkextractors_deprecated.py -tests/test_proxy_connect.py - scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index 431428331..75c74daf5 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -13,6 +13,7 @@ error = KeyError _DATABASES = collections.defaultdict(DummyDB) + def open(file, flag='r', mode=0o666): """Open or create a dummy database compatible. diff --git a/tests/mockserver.py b/tests/mockserver.py index b766bb653..fe28176d4 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,7 +1,9 @@ -from __future__ import print_function -import sys, time, random, os, json -from six.moves.urllib.parse import urlencode +import json +import os +import random +import sys from subprocess import Popen, PIPE +from urllib.parse import urlencode from OpenSSL import SSL from twisted.web.server import Site, NOT_DONE_YET diff --git a/tests/pipelines.py b/tests/pipelines.py index 7e2895a5c..d7d3b5259 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -2,6 +2,7 @@ Some pipelines used for testing """ + class ZeroDivisionErrorPipeline(object): def open_spider(self, spider): diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 2e8d319d2..e9bf310b9 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,5 +1,7 @@ # Tests requirements jmespath +mitmproxy; python_version >= '3.6' +mitmproxy<4.0.0; python_version < '3.6' pytest pytest-cov pytest-twisted diff --git a/tests/spiders.py b/tests/spiders.py index 7816bf7c7..981bd2eb8 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking """ import time -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode from scrapy.spiders import Spider from scrapy.http import Request @@ -16,6 +16,7 @@ class MockServerSpider(Spider): super(MockServerSpider, self).__init__(*args, **kwargs) self.mockserver = mockserver + class MetaSpider(MockServerSpider): name = 'meta' diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py index 56cfe642a..da99a6be8 100644 --- a/tests/test_cmdline/__init__.py +++ b/tests/test_cmdline/__init__.py @@ -1,13 +1,12 @@ -from io import StringIO import json import os import pstats import shutil -import six -from subprocess import Popen, PIPE import sys import tempfile import unittest +from io import StringIO +from subprocess import Popen, PIPE from scrapy.utils.test import get_testenv @@ -26,17 +25,15 @@ class CmdlineTest(unittest.TestCase): return comm.decode(encoding) def test_default_settings(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ - 'default') + self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default') def test_override_settings_using_set_arg(self): - self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), \ - 'override') + self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', + 'TEST1=override'), 'override') def test_override_settings_using_envvar(self): self.env['SCRAPY_TEST1'] = 'override' - self.assertEqual(self._execute('settings', '--get', 'TEST1'), \ - 'override') + self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override') def test_profiling(self): path = tempfile.mkdtemp() @@ -65,5 +62,5 @@ class CmdlineTest(unittest.TestCase): for char in ("'", "<", ">", 'u"'): settingsstr = settingsstr.replace(char, '"') settingsdict = json.loads(settingsstr) - six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys()) + self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys()) self.assertEqual(200, settingsdict[EXT_PATH]) diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py index 28456b55d..c64e87d81 100644 --- a/tests/test_cmdline/extensions.py +++ b/tests/test_cmdline/extensions.py @@ -1,5 +1,6 @@ """A test extension used to check the settings loading order""" + class TestExtension(object): def __init__(self, settings): diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 3fa3ed930..9d3c8fe73 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) - out = out.replace(b'\r', b'') # required on win32 + out = out.replace(b'\r', b'') # required on win32 assert b'Server: TwistedWeb' in out, out assert b'Content-Type: text/plain' in out diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index b134beb88..b7035fdff 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -12,6 +12,7 @@ def _textmode(bstr): and reading from it in text mode""" return to_unicode(bstr).replace(os.linesep, '\n') + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b2e358700..582e3d052 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,6 +1,5 @@ from unittest import TextTestResult -from six import get_unbound_function from twisted.internet import defer from twisted.python import failure from twisted.trial import unittest @@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase): with MockServer() as mockserver: contract_doc = '@url {}'.format(mockserver.url('/status?n=200')) - get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc - get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc + TestSameUrlSpider.parse_first.__doc__ = contract_doc + TestSameUrlSpider.parse_second.__doc__ = contract_doc crawler = CrawlerRunner().create_crawler(TestSameUrlSpider) yield crawler.crawl() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3fc13eeb7..3307899b7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -140,7 +140,7 @@ class CrawlTestCase(TestCase): def test_unbounded_response(self): # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" - from six.moves.urllib.parse import urlencode + from urllib.parse import urlencode query = urlencode({'raw': '''\ HTTP/1.1 200 OK Server: Apache-Coyote/1.1 diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8eb2389e2..e37a2ff0e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,4 +1,7 @@ import logging +import os +import subprocess +import sys import warnings from twisted.internet import defer @@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.misc import load_object from scrapy.extensions.throttle import AutoThrottle from scrapy.extensions import telnet +from scrapy.utils.test import get_testenv class BaseCrawlerTest(unittest.TestCase): @@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase): yield runner.crawl(NoRequestsSpider) self.assertEqual(runner.bootstrap_failed, True) + + +class CrawlerProcessSubprocess(unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + + def run_script(self, script_name): + script_path = os.path.join(self.script_dir, script_name) + args = (sys.executable, script_path) + p = subprocess.Popen(args, env=get_testenv(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + return stderr.decode('utf-8') + + def test_simple(self): + log = self.run_script('simple.py') + self.assertIn('Spider closed (finished)', log) diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 03bf2ffcf..e31ccd9b5 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -1,6 +1,7 @@ from importlib import import_module from twisted.trial import unittest + class ScrapyUtilsTest(unittest.TestCase): def test_required_openssl_version(self): try: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 59d4a3eec..ce39f8545 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -1,5 +1,4 @@ import os -import six import shutil import tempfile from unittest import mock @@ -34,7 +33,7 @@ from scrapy.responsetypes import responsetypes from scrapy.settings import Settings from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from tests.mockserver import MockServer, ssl_context_factory, Echo from tests.spiders import SingleRequestSpider @@ -350,7 +349,7 @@ class HttpTestCase(unittest.TestCase): return self.download_request(request, Spider('foo')).addCallback(_test) def test_payload(self): - body = b'1'*100 # PayloadResource requires body length to be 100 + body = b'1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) d = self.download_request(request, Spider('foo')) d.addCallback(lambda r: r.body) @@ -543,7 +542,7 @@ class Https11InvalidDNSPattern(Https11TestCase): def setUp(self): try: - from service_identity.exceptions import CertificateError + from service_identity.exceptions import CertificateError # noqa: F401 except ImportError: raise unittest.SkipTest("cryptography lib is too old") self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' @@ -630,18 +629,16 @@ class Http11MockServerTestCase(unittest.TestCase): # download_maxsize < 100, hence the CancelledError self.assertIsInstance(failure.value, defer.CancelledError) - if six.PY2: - request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') - request = request.replace(url=self.mockserver.url('/xpayload')) - yield crawler.crawl(seed=request) - # download_maxsize = 50 is enough for the gzipped response - failure = crawler.spider.meta.get('failure') - self.assertTrue(failure is None) - reason = crawler.spider.meta['close_reason'] - self.assertTrue(reason, 'finished') - else: - # See issue https://twistedmatrix.com/trac/ticket/8175 - raise unittest.SkipTest("xpayload only enabled for PY2") + # See issue https://twistedmatrix.com/trac/ticket/8175 + raise unittest.SkipTest("xpayload fails on PY3") + request.headers.setdefault(b'Accept-Encoding', b'gzip,deflate') + request = request.replace(url=self.mockserver.url('/xpayload')) + yield crawler.crawl(seed=request) + # download_maxsize = 50 is enough for the gzipped response + failure = crawler.spider.meta.get('failure') + self.assertTrue(failure is None) + reason = crawler.spider.meta['close_reason'] + self.assertTrue(reason, 'finished') class UriResource(resource.Resource): @@ -698,7 +695,9 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return self.download_request(request, Spider('foo')).addCallback(_test) + with self.assertWarnsRegex(ScrapyDeprecationWarning, + r'Using HTTPS proxies in the noconnect mode is deprecated'): + return self.download_request(request, Spider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -713,6 +712,9 @@ class HttpProxyTestCase(unittest.TestCase): class Http10ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP10DownloadHandler + def test_download_with_proxy_https_noconnect(self): + raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler') + class Http11ProxyTestCase(HttpProxyTestCase): download_handler_cls = HTTP11DownloadHandler @@ -778,7 +780,7 @@ class S3TestCase(unittest.TestCase): @contextlib.contextmanager def _mocked_date(self, date): try: - import botocore.auth + import botocore.auth # noqa: F401 except ImportError: yield else: @@ -803,8 +805,8 @@ class S3TestCase(unittest.TestCase): req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') def test_request_signing2(self): # puts an object into the johnsmith bucket. @@ -816,21 +818,22 @@ class S3TestCase(unittest.TestCase): }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') def test_request_signing3(self): # lists the content of the johnsmith bucket. date = 'Tue, 27 Mar 2007 19:42:41 +0000' - req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \ - method='GET', headers={ - 'User-Agent': 'Mozilla/5.0', - 'Date': date, - }) + req = Request( + 's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', + method='GET', headers={ + 'User-Agent': 'Mozilla/5.0', + 'Date': date, + }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') def test_request_signing4(self): # fetches the access control policy sub-resource for the 'johnsmith' bucket. @@ -839,23 +842,25 @@ class S3TestCase(unittest.TestCase): method='GET', headers={'Date': date}) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') def test_request_signing5(self): - try: import botocore - except ImportError: pass + try: + import botocore # noqa: F401 + except ImportError: + pass else: raise unittest.SkipTest( 'botocore does not support overriding date with x-amz-date') # deletes an object from the 'johnsmith' bucket using the # path-style and Date alternative. date = 'Tue, 27 Mar 2007 21:20:27 +0000' - req = Request('s3://johnsmith/photos/puppy.jpg', \ - method='DELETE', headers={ - 'Date': date, - 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', - }) + req = Request( + 's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={ + 'Date': date, + 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', + }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) # botocore does not override Date with x-amz-date @@ -865,25 +870,26 @@ class S3TestCase(unittest.TestCase): def test_request_signing6(self): # uploads an object to a CNAME style virtual hosted bucket with metadata. date = 'Tue, 27 Mar 2007 21:06:08 +0000' - req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \ - method='PUT', headers={ - 'User-Agent': 'curl/7.15.5', - 'Host': 'static.johnsmith.net:8080', - 'Date': date, - 'x-amz-acl': 'public-read', - 'content-type': 'application/x-download', - 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', - 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', - 'X-Amz-Meta-FileChecksum': '0x02661779', - 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', - 'Content-Disposition': 'attachment; filename=database.dat', - 'Content-Encoding': 'gzip', - 'Content-Length': '5913339', - }) + req = Request( + 's3://static.johnsmith.net:8080/db-backup.dat.gz', + method='PUT', headers={ + 'User-Agent': 'curl/7.15.5', + 'Host': 'static.johnsmith.net:8080', + 'Date': date, + 'x-amz-acl': 'public-read', + 'content-type': 'application/x-download', + 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', + 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', + 'X-Amz-Meta-FileChecksum': '0x02661779', + 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', + 'Content-Disposition': 'attachment; filename=database.dat', + 'Content-Encoding': 'gzip', + 'Content-Length': '5913339', + }) with self._mocked_date(date): httpreq = self.download_request(req, self.spider) - self.assertEqual(httpreq.headers['Authorization'], \ - b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') + self.assertEqual(httpreq.headers['Authorization'], + b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=') def test_request_signing7(self): # ensure that spaces are quoted properly before signing diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py index 493691ea4..5a56c9db2 100644 --- a/tests/test_downloadermiddleware_ajaxcrawlable.py +++ b/tests/test_downloadermiddleware_ajaxcrawlable.py @@ -5,8 +5,10 @@ from scrapy.spiders import Spider from scrapy.http import Request, HtmlResponse, Response from scrapy.utils.test import get_crawler + __doctests__ = ['scrapy.downloadermiddlewares.ajaxcrawl'] + class AjaxCrawlMiddlewareTest(unittest.TestCase): def setUp(self): crawler = get_crawler(Spider, {'AJAXCRAWL_ENABLED': True}) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 9d863b6e3..9401dd66d 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -1,12 +1,9 @@ -from __future__ import print_function import time import tempfile import shutil import unittest import email.utils from contextlib import contextmanager -import pytest -import sys from scrapy.http import Response, HtmlResponse, Request from scrapy.spiders import Spider @@ -149,6 +146,7 @@ class FilesystemStorageTest(DefaultStorageTest): storage_class = 'scrapy.extensions.httpcache.FilesystemCacheStorage' + class FilesystemStorageGzipTest(FilesystemStorageTest): def _get_settings(self, **new_settings): diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 0745c8dd3..c6a823b53 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -70,7 +70,7 @@ class HttpCompressionTest(TestCase): def test_process_response_br(self): try: - import brotli + import brotli # noqa: F401 except ImportError: raise SkipTest("no brotli") response = self._getresponse('br') diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 30920b2da..36743b1de 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -1,11 +1,10 @@ import os -import sys from functools import partial -from twisted.trial.unittest import TestCase, SkipTest +from twisted.trial.unittest import TestCase from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured -from scrapy.http import Response, Request +from scrapy.http import Request from scrapy.spiders import Spider from scrapy.crawler import Crawler from scrapy.settings import Settings diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 51b79b6c3..e09d66086 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -165,12 +165,12 @@ class MaxRetryTimesTest(unittest.TestCase): # SETTINGS: meta(max_retry_times) = 4 meta_max_retry_times = 4 - req = Request(self.invalid_url, meta= \ - {'max_retry_times': meta_max_retry_times, 'dont_retry': True}) + req = Request(self.invalid_url, meta={ + 'max_retry_times': meta_max_retry_times, 'dont_retry': True + }) self._test_retry(req, DNSLookupError('foo'), 0) - def _test_retry(self, req, exception, max_retry_times): for i in range(0, max_retry_times): diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 8266bf35f..a1645ed96 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import - from unittest import mock from twisted.internet import reactor, error diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index d7eb98c97..e4b0bdf83 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -12,6 +12,7 @@ from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider + class FromCrawlerRFPDupeFilter(RFPDupeFilter): @classmethod diff --git a/tests/test_engine.py b/tests/test_engine.py index 30150391a..a48b63025 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -10,9 +10,10 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ -from __future__ import print_function -import sys, os, re -from six.moves.urllib.parse import urlparse +import os +import re +import sys +from urllib.parse import urlparse from twisted.internet import reactor, defer from twisted.web import server, static, util @@ -90,8 +91,8 @@ def start_test_site(debug=False): port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1") if debug: - print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \ - % port.getHost().port) + print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." + % port.getHost().port) return port @@ -270,7 +271,6 @@ class EngineTest(unittest.TestCase): self.run.signals_catched[signals.spider_opened]) self.assertEqual({'spider': self.run.spider}, self.run.signals_catched[signals.spider_idle]) - self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17 self.assertEqual({'spider': self.run.spider, 'reason': 'finished'}, self.run.signals_catched[signals.spider_closed]) diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 0046c5666..5d1f5c182 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -1,15 +1,13 @@ -from __future__ import absolute_import import re import json import marshal +import pickle import tempfile import unittest from io import BytesIO from datetime import datetime -from six.moves import cPickle as pickle import lxml.etree -import six from scrapy.item import Item, Field from scrapy.utils.python import to_unicode @@ -80,7 +78,7 @@ class BaseItemExporterTest(unittest.TestCase): ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1') _, name = list(ie._get_serialized_fields(self.i))[0] - assert isinstance(name, six.text_type) + assert isinstance(name, str) self.assertEqual(name, u'John\xa3') def test_field_custom_serializer(self): diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 875ceb83c..873a97248 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -3,7 +3,7 @@ from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials from twisted.internet import defer -from scrapy.extensions.telnet import TelnetConsole, logger +from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 87139e81f..2ca57c19d 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -1,15 +1,15 @@ -from __future__ import absolute_import import os import csv import json import warnings -from io import BytesIO import tempfile import shutil import string +from io import BytesIO +from pathlib import Path from unittest import mock -from six.moves.urllib.parse import urljoin, urlparse, quote -from six.moves.urllib.request import pathname2url +from urllib.parse import urljoin, urlparse, quote +from urllib.request import pathname2url from zope.interface.verify import verifyObject from twisted.trial import unittest @@ -28,8 +28,6 @@ from scrapy.extensions.feedexport import ( from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler from scrapy.utils.python import to_unicode -from pathlib import Path - class FileFeedStorageTest(unittest.TestCase): @@ -167,7 +165,7 @@ class S3FeedStorageTest(unittest.TestCase): create=True) def test_parse_credentials(self): try: - import boto + import boto # noqa: F401 except ImportError: raise unittest.SkipTest("S3FeedStorage requires boto") aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', @@ -268,7 +266,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): try: - import botocore + import botocore # noqa: F401 except ImportError: raise unittest.SkipTest('botocore is required') @@ -288,7 +286,7 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): try: - import botocore + import botocore # noqa: F401 except ImportError: raise unittest.SkipTest('botocore is required') diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py index 0a9ed500a..45ddb42ba 100644 --- a/tests/test_http_cookies.py +++ b/tests/test_http_cookies.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from unittest import TestCase from scrapy.http import Request, Response diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py index 69d906fbf..cf3fc8496 100644 --- a/tests/test_http_headers.py +++ b/tests/test_http_headers.py @@ -3,6 +3,7 @@ import copy from scrapy.http import Headers + class HeadersTest(unittest.TestCase): def assertSortedEqual(self, first, second, msg=None): @@ -85,9 +86,6 @@ class HeadersTest(unittest.TestCase): self.assertSortedEqual(h.items(), [(b'X-Forwarded-For', [b'ip1', b'ip2']), (b'Content-Type', [b'text/html'])]) - self.assertSortedEqual(h.iteritems(), - [(b'X-Forwarded-For', [b'ip1', b'ip2']), - (b'Content-Type', [b'text/html'])]) self.assertSortedEqual(h.values(), [b'ip2', b'text/html']) def test_update(self): diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 5b1c8578e..e30417b30 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -1,14 +1,10 @@ -# -*- coding: utf-8 -*- -import cgi import unittest import re import json -from unittest import mock -from urllib.parse import unquote_to_bytes +import xmlrpc.client import warnings - -from six.moves import xmlrpc_client as xmlrpclib -from six.moves.urllib.parse import urlparse, parse_qs, unquote +from unittest import mock +from urllib.parse import parse_qs, unquote_to_bytes, urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse from scrapy.utils.python import to_bytes, to_unicode @@ -49,6 +45,8 @@ class RequestTest(unittest.TestCase): def test_url_no_scheme(self): self.assertRaises(ValueError, self.request_class, 'foo') + self.assertRaises(ValueError, self.request_class, '/foo/') + self.assertRaises(ValueError, self.request_class, '/foo:bar') def test_headers(self): # Different ways of setting headers attribute @@ -64,7 +62,7 @@ class RequestTest(unittest.TestCase): # headers must not be unicode h = Headers({'key1': u'val1', u'key2': 'val2'}) h[u'newkey'] = u'newval' - for k, v in h.iteritems(): + for k, v in h.items(): self.assertIsInstance(k, bytes) for s in v: self.assertIsInstance(s, bytes) @@ -151,7 +149,7 @@ class RequestTest(unittest.TestCase): r2 = self.request_class(url="http://www.example.com/", body=b"") assert isinstance(r2.body, bytes) - self.assertEqual(r2.encoding, 'utf-8') # default encoding + self.assertEqual(r2.encoding, 'utf-8') # default encoding r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') assert isinstance(r3.body, bytes) @@ -624,8 +622,9 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, formdata={'two': '2'}, \ - clickdata={'name': 'clickable2'}) + req = self.request_class.from_response( + response, formdata={'two': '2'}, clickdata={'name': 'clickable2'} + ) fs = _qs(req) self.assertEqual(fs[b'clickable2'], [b'clicked2']) self.assertFalse(b'clickable1' in fs, fs) @@ -673,8 +672,9 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, \ - clickdata={u'name': u'clickable', u'value': u'clicked2'}) + req = self.request_class.from_response( + response, clickdata={u'name': u'clickable', u'value': u'clicked2'} + ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) self.assertEqual(fs[b'one'], [b'clicked1']) @@ -688,8 +688,9 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, \ - clickdata={u'name': u'price in \u00a3'}) + req = self.request_class.from_response( + response, clickdata={u'name': u'price in \u00a3'} + ) fs = _qs(req, to_unicode=True) self.assertTrue(fs[u'price in \u00a3']) @@ -702,8 +703,9 @@ class FormRequestTest(RequestTest): """, encoding='latin1') - req = self.request_class.from_response(response, \ - clickdata={u'name': u'price in \u00a5'}) + req = self.request_class.from_response( + response, clickdata={u'name': u'price in \u00a5'} + ) fs = _qs(req, to_unicode=True, encoding='latin1') self.assertTrue(fs[u'price in \u00a5']) @@ -718,8 +720,9 @@ class FormRequestTest(RequestTest): """) - req = self.request_class.from_response(response, formname='form2', \ - clickdata={u'name': u'clickable'}) + req = self.request_class.from_response( + response, formname='form2', clickdata={u'name': u'clickable'} + ) fs = _qs(req) self.assertEqual(fs[b'clickable'], [b'clicked2']) self.assertEqual(fs[b'field2'], [b'value2']) @@ -727,8 +730,9 @@ class FormRequestTest(RequestTest): def test_from_response_override_clickable(self): response = _buildresponse('''
''') - req = self.request_class.from_response(response, \ - formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}) + req = self.request_class.from_response( + response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'} + ) fs = _qs(req) self.assertEqual(fs[b'clickme'], [b'two']) @@ -855,7 +859,7 @@ class FormRequestTest(RequestTest):
""") - self.assertRaises(IndexError, self.request_class.from_response, \ + self.assertRaises(IndexError, self.request_class.from_response, response, formname="form3", formnumber=2) def test_from_response_formid_exists(self): @@ -909,7 +913,7 @@ class FormRequestTest(RequestTest):
""") - self.assertRaises(IndexError, self.request_class.from_response, \ + self.assertRaises(IndexError, self.request_class.from_response, response, formid="form3", formnumber=2) def test_from_response_select(self): @@ -1220,7 +1224,7 @@ class XmlRpcRequestTest(RequestTest): r = self.request_class('http://scrapytest.org/rpc2', **kwargs) self.assertEqual(r.headers[b'Content-Type'], b'text/xml') self.assertEqual(r.body, - to_bytes(xmlrpclib.dumps(**kwargs), + to_bytes(xmlrpc.client.dumps(**kwargs), encoding=kwargs.get('encoding', 'utf-8'))) self.assertEqual(r.method, 'POST') self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8')) diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 883c943da..960ecea3e 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import unittest -import six from w3lib.encoding import resolve_encoding from scrapy.http import (Request, Response, TextResponse, HtmlResponse, @@ -102,7 +101,7 @@ class BaseResponseTest(unittest.TestCase): self.assertEqual(r4.flags, []) def _assert_response_values(self, response, encoding, body): - if isinstance(body, six.text_type): + if isinstance(body, str): body_unicode = body body_bytes = body.encode(encoding) else: @@ -110,7 +109,7 @@ class BaseResponseTest(unittest.TestCase): body_bytes = body assert isinstance(response.body, bytes) - assert isinstance(response.text, six.text_type) + assert isinstance(response.text, str) self._assert_response_encoding(response, encoding) self.assertEqual(response.body, body_bytes) self.assertEqual(response.body_as_unicode(), body_unicode) @@ -220,11 +219,11 @@ class TextResponseTest(BaseResponseTest): r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251') # check body_as_unicode - self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type)) + self.assertTrue(isinstance(r1.body_as_unicode(), str)) self.assertEqual(r1.body_as_unicode(), unicode_string) # check response.text - self.assertTrue(isinstance(r1.text, six.text_type)) + self.assertTrue(isinstance(r1.text, str)) self.assertEqual(r1.text, unicode_string) def test_encoding(self): @@ -317,8 +316,8 @@ class TextResponseTest(BaseResponseTest): assert u'SUFFIX' in r.text, repr(r.text) # Do not destroy html tags due to encoding bugs - r = self.response_class("http://example.com", encoding='utf-8', \ - body=b'\xf0value') + r = self.response_class("http://example.com", encoding='utf-8', + body=b'\xf0value') assert u'value' in r.text, repr(r.text) # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse diff --git a/tests/test_item.py b/tests/test_item.py index 49117ef04..30463a0f5 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -3,8 +3,6 @@ import unittest from unittest import mock from warnings import catch_warnings -import six - from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta @@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): - class MyItem(six.with_metaclass(ItemMeta, Item)): + class MyItem(Item, metaclass=ItemMeta): def __init__(self, *args, **kwargs): # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: diff --git a/tests/test_linkextractors_deprecated.py b/tests/test_linkextractors_deprecated.py deleted file mode 100644 index 1366971be..000000000 --- a/tests/test_linkextractors_deprecated.py +++ /dev/null @@ -1,233 +0,0 @@ -# -*- coding: utf-8 -*- -import unittest -from scrapy.linkextractors.regex import RegexLinkExtractor -from scrapy.http import HtmlResponse -from scrapy.link import Link -from scrapy.linkextractors.htmlparser import HtmlParserLinkExtractor -from scrapy.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor -from tests import get_testdata - -from tests.test_linkextractors import Base - - -class BaseSgmlLinkExtractorTestCase(unittest.TestCase): - # XXX: should we move some of these tests to base link extractor tests? - - def test_basic(self): - html = """Page title<title> - <body><p><a href="item/12.html">Item 12</a></p> - <p><a href="/about.html">About us</a></p> - <img src="/logo.png" alt="Company logo (not a link)" /> - <p><a href="../othercat.html">Other category</a></p> - <p><a href="/">>></a></p> - <p><a href="/" /></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), - Link(url='http://example.org/about.html', text='About us'), - Link(url='http://example.org/othercat.html', text='Other category'), - Link(url='http://example.org/', text='>>'), - Link(url='http://example.org/', text='')]) - - def test_base_url(self): - html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" /> - <body><p><a href="item/12.html">Item 12</a></p> - </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", body=html) - - lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href - self.assertEqual(lx.extract_links(response), - [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) - - # base url is an absolute path and relative to host - html = """<html><head><title>Page title<title><base href="/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://example.org/item/12.html', text='Item 12')]) - - # base url has no scheme - html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> - <body><p><a href="item/12.html">Item 12</a></p></body></html>""" - response = HtmlResponse("https://example.org/somepage/index.html", body=html) - self.assertEqual(lx.extract_links(response), - [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) - - def test_link_text_wrong_encoding(self): - html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>""" - response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response), [ - Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'), - ]) - - def test_extraction_encoding(self): - body = get_testdata('link_extractor', 'linkextractor_noenc.html') - response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) - response_noenc = HtmlResponse(url='http://example.com/noenc', body=body) - body = get_testdata('link_extractor', 'linkextractor_latin1.html') - response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) - - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response_utf8), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) - - self.assertEqual(lx.extract_links(response_noenc), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), - ]) - - # document encoding does not affect URL path component, only query part - # >>> u'sample_ñ.html'.encode('utf8') - # b'sample_\xc3\xb1.html' - # >>> u"sample_á.html".encode('utf8') - # b'sample_\xc3\xa1.html' - # >>> u"sample_ö.html".encode('utf8') - # b'sample_\xc3\xb6.html' - # >>> u"£32".encode('latin1') - # b'\xa332' - # >>> u"µ".encode('latin1') - # b'\xb5' - self.assertEqual(lx.extract_links(response_latin1), [ - Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')), - Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''), - ]) - - def test_matches(self): - url1 = 'http://lotsofstuff.com/stuff1/index' - url2 = 'http://evenmorestuff.com/uglystuff/index' - - lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.matches(url1), True) - self.assertEqual(lx.matches(url2), True) - - -class HtmlParserLinkExtractorTestCase(unittest.TestCase): - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = HtmlParserLinkExtractor() - self.assertEqual(lx.extract_links(self.response), [ - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'), - Link(url='http://example.com/page%204.html', text=u'href with whitespaces'), - ]) - - def test_link_wrong_href(self): - html = """ - <a href="http://example.org/item1.html">Item 1</a> - <a href="http://[example.org/item2.html">Item 2</a> - <a href="http://example.org/item3.html">Item 3</a> - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = HtmlParserLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - -class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase): - extractor_cls = SgmlLinkExtractor - escapes_whitespace = True - - def test_deny_extensions(self): - html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" - response = HtmlResponse("http://example.org/", body=html) - lx = SgmlLinkExtractor(deny_extensions="jpg") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.org/page.html', text=u'asd'), - ]) - - def test_attrs_sgml(self): - html = """<html><area href="sample1.html"></area> - <a ref="sample2.html">sample text 2</a></html>""" - response = HtmlResponse("http://example.com/index.html", body=html) - lx = SgmlLinkExtractor(attrs="href") - self.assertEqual(lx.extract_links(response), [ - Link(url='http://example.com/sample1.html', text=u''), - ]) - - def test_link_nofollow(self): - html = """ - <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a> - <a href="about.html">About us</a> - <a href="http://google.com/something" rel="external nofollow">Something</a> - """ - response = HtmlResponse("http://example.org/page.html", body=html) - lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), - Link(url='http://example.org/about.html', text=u'About us', nofollow=False), - Link(url='http://google.com/something', text=u'Something', nofollow=True), - ]) - - -class RegexLinkExtractorTestCase(unittest.TestCase): - # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten - # not to depend on SgmlLinkExractor. Its speed is also much worse - # than it should be. - - def setUp(self): - body = get_testdata('link_extractor', 'sgml_linkextractor.html') - self.response = HtmlResponse(url='http://example.com/index', body=body) - - def test_extraction(self): - # Default arguments - lx = RegexLinkExtractor() - self.assertEqual(lx.extract_links(self.response), - [Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'), - Link(url='http://www.google.com/something', text=u''), - Link(url='http://example.com/innertag.html', text=u'inner tag'),]) - - def test_link_wrong_href(self): - html = """ - <a href="http://example.org/item1.html">Item 1</a> - <a href="http://[example.org/item2.html">Item 2</a> - <a href="http://example.org/item3.html">Item 3</a> - """ - response = HtmlResponse("http://example.org/index.html", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False), - Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False), - ]) - - def test_html_base_href(self): - html = """ - <html> - <head> - <base href="http://b.com/"> - </head> - <body> - <a href="test.html"></a> - </body> - </html> - """ - response = HtmlResponse("http://a.com/", body=html) - lx = RegexLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], [ - Link(url='http://b.com/test.html', text=u'', nofollow=False), - ]) - - @unittest.expectedFailure - def test_extraction(self): - # RegexLinkExtractor doesn't parse URLs with leading/trailing - # whitespaces correctly. - super(RegexLinkExtractorTestCase, self).test_extraction() diff --git a/tests/test_loader.py b/tests/test_loader.py index b87602809..579a85ff6 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -1,8 +1,6 @@ from functools import partial import unittest -import six - from scrapy.http import HtmlResponse from scrapy.item import Item, Field from scrapy.loader import ItemLoader @@ -157,7 +155,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_get_value(self): il = NameItemLoader() - self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper)) + self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper)) self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$')) self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$')) @@ -258,7 +256,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_custom_input_processors(self): class ChildItemLoader(TestItemLoader): - name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase) + name_in = MapCompose(TestItemLoader.name_in, str.swapcase) il = ChildItemLoader() il.add_value('name', u'marta') @@ -266,7 +264,7 @@ class BasicItemLoaderTest(unittest.TestCase): def test_extend_default_input_processors(self): class ChildDefaultedItemLoader(DefaultedItemLoader): - name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase) + name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase) il = ChildDefaultedItemLoader() il.add_value('name', u'marta') @@ -689,7 +687,7 @@ class ProcessorsTest(unittest.TestCase): self.assertRaises(TypeError, proc, [None, '', 'hello', 'world']) self.assertEqual(proc(['', 'hello', 'world']), u' hello world') self.assertEqual(proc(['hello', 'world']), u'hello world') - self.assertIsInstance(proc(['hello', 'world']), six.text_type) + self.assertIsInstance(proc(['hello', 'world']), str) def test_compose(self): proc = Compose(lambda v: v[0], str.upper) @@ -704,12 +702,12 @@ class ProcessorsTest(unittest.TestCase): def test_mapcompose(self): def filter_world(x): return None if x == 'world' else x - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']), [u'HELLO', u'THIS', u'IS', u'SCRAPY']) - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertEqual(proc(None), []) - proc = MapCompose(filter_world, six.text_type.upper) + proc = MapCompose(filter_world, str.upper) self.assertRaises(ValueError, proc, [1]) proc = MapCompose(filter_world, lambda x: x + 1) self.assertRaises(ValueError, proc, 'hello') @@ -994,5 +992,53 @@ class SelectJmesTestCase(unittest.TestCase): ) +# Functions as processors + +def function_processor_strip(iterable): + return [x.strip() for x in iterable] + + +def function_processor_upper(iterable): + return [x.upper() for x in iterable] + + +class FunctionProcessorItem(Item): + foo = Field( + input_processor=function_processor_strip, + output_processor=function_processor_upper, + ) + + +class FunctionProcessorItemLoader(ItemLoader): + default_item_class = FunctionProcessorItem + + +class FunctionProcessorDictLoader(ItemLoader): + default_item_class = dict + foo_in = function_processor_strip + foo_out = function_processor_upper + + +class FunctionProcessorTestCase(unittest.TestCase): + + def test_processor_defined_in_item(self): + lo = FunctionProcessorItemLoader() + lo.add_value('foo', ' bar ') + lo.add_value('foo', [' asdf ', ' qwerty ']) + self.assertEqual( + dict(lo.load_item()), + {'foo': ['BAR', 'ASDF', 'QWERTY']} + ) + + def test_processor_defined_in_item_loader(self): + lo = FunctionProcessorDictLoader() + lo.add_value('foo', ' bar ') + lo.add_value('foo', [' asdf ', ' qwerty ']) + self.assertEqual( + dict(lo.load_item()), + {'foo': ['BAR', 'ASDF', 'QWERTY']} + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index b4ea30bb7..7d8c6ec7f 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -3,7 +3,6 @@ import unittest from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase as TwistedTestCase -import six from scrapy.crawler import CrawlerRunner from scrapy.exceptions import DropItem @@ -23,13 +22,13 @@ class CustomItem(Item): return "name: %s" % self['name'] -class LoggingContribTest(unittest.TestCase): +class LogFormatterTestCase(unittest.TestCase): def setUp(self): self.formatter = LogFormatter() self.spider = Spider('default') - def test_crawled(self): + def test_crawled_with_referer(self): req = Request("http://www.example.com") res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) @@ -37,6 +36,7 @@ class LoggingContribTest(unittest.TestCase): self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)") + def test_crawled_without_referer(self): req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) res = Response("http://www.example.com", flags=['cached']) logkws = self.formatter.crawled(req, res, self.spider) @@ -59,9 +59,19 @@ class LoggingContribTest(unittest.TestCase): logkws = self.formatter.dropped(item, exception, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() - assert all(isinstance(x, six.text_type) for x in lines) + assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Dropped: \u2018", '{}']) + def test_error(self): + # In practice, the complete traceback is shown by passing the + # 'exc_info' argument to the logging function + item = {'key': 'value'} + exception = Exception() + response = Response("http://www.example.com") + logkws = self.formatter.error(item, exception, response, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, u"'Error processing {'key': 'value'}'") + def test_scraped(self): item = CustomItem() item['name'] = u'\xa3' @@ -69,32 +79,52 @@ class LoggingContribTest(unittest.TestCase): logkws = self.formatter.scraped(item, response, self.spider) logline = logkws['msg'] % logkws['args'] lines = logline.splitlines() - assert all(isinstance(x, six.text_type) for x in lines) + assert all(isinstance(x, str) for x in lines) self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3']) class LogFormatterSubclass(LogFormatter): def crawled(self, request, response, spider): - kwargs = super(LogFormatterSubclass, self).crawled( - request, response, spider) + kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider) CRAWLEDMSG = ( - u"Crawled (%(status)s) %(request)s (referer: " - u"%(referer)s)%(flags)s" + u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s" ) + log_args = kwargs['args'] + log_args['flags'] = str(request.flags) return { 'level': kwargs['level'], 'msg': CRAWLEDMSG, - 'args': kwargs['args'] + 'args': log_args, } -class LogformatterSubclassTest(LoggingContribTest): +class LogformatterSubclassTest(LogFormatterTestCase): def setUp(self): self.formatter = LogFormatterSubclass() self.spider = Spider('default') + def test_crawled_with_referer(self): + req = Request("http://www.example.com") + res = Response("http://www.example.com") + logkws = self.formatter.crawled(req, res, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, + "Crawled (200) <GET http://www.example.com> (referer: None) []") + + def test_crawled_without_referer(self): + req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached']) + res = Response("http://www.example.com") + logkws = self.formatter.crawled(req, res, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, + "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']") + def test_flags_in_request(self): - pass + req = Request("http://www.example.com", flags=['test', 'flag']) + res = Response("http://www.example.com") + logkws = self.formatter.crawled(req, res, self.spider) + logline = logkws['msg'] % logkws['args'] + self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']") class SkipMessagesLogFormatter(LogFormatter): @@ -118,6 +148,7 @@ class DropSomeItemsPipeline(object): else: self.drop = True + class ShowOrSkipMessagesTestCase(TwistedTestCase): def setUp(self): self.mockserver = MockServer() diff --git a/tests/test_mail.py b/tests/test_mail.py index b139e98d8..ddb0f1e70 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -6,6 +6,7 @@ from email.charset import Charset from scrapy.mail import MailSender + class MailSenderTest(unittest.TestCase): def test_send(self): diff --git a/tests/test_middleware.py b/tests/test_middleware.py index af9b43d61..ebf817c7e 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -4,6 +4,7 @@ from scrapy.settings import Settings from scrapy.exceptions import NotConfigured from scrapy.middleware import MiddlewareManager + class M1(object): def open_spider(self, spider): @@ -15,6 +16,7 @@ class M1(object): def process(self, response, request, spider): pass + class M2(object): def open_spider(self, spider): @@ -25,6 +27,7 @@ class M2(object): pass + class M3(object): def process(self, response, request, spider): @@ -54,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process'): self.methods['process'].append(mw.process) + class MiddlewareManagerTest(unittest.TestCase): def test_init(self): diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index bd40e4103..52f2b554e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -1,11 +1,11 @@ import os import random import time +from io import BytesIO from tempfile import mkdtemp from shutil import rmtree from unittest import mock -from six.moves.urllib.parse import urlparse -from six import BytesIO +from urllib.parse import urlparse from twisted.trial import unittest from twisted.internet import defer diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 4f7265763..7f1cb4a11 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -1,7 +1,6 @@ import io import hashlib import random -import warnings from tempfile import mkdtemp from shutil import rmtree diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index ad958e25f..1fcc5799e 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,5 +1,3 @@ -from __future__ import print_function - from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure @@ -240,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)]) m = self.pipe._mockcalled # only once - self.assertEqual(m[0], 'get_media_requests') # first hook called + self.assertEqual(m[0], 'get_media_requests') # first hook called self.assertEqual(m.count('get_media_requests'), 1) self.assertEqual(m.count('item_completed'), 1) - self.assertEqual(m[-1], 'item_completed') # last hook called + self.assertEqual(m[-1], 'item_completed') # last hook called # twice, one per request self.assertEqual(m.count('media_to_download'), 2) # one to handle success and other for failure @@ -254,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): def test_get_media_requests(self): # returns single Request (without callback) req = Request('http://url') - item = dict(requests=req) # pass a single item + item = dict(requests=req) # pass a single item new_item = yield self.pipe.process_item(item, self.spider) assert new_item is item assert request_fingerprint(req) in self.info.downloaded diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py new file mode 100644 index 000000000..bc53f5427 --- /dev/null +++ b/tests/test_pipelines.py @@ -0,0 +1,71 @@ +from twisted.internet import defer +from twisted.internet.defer import Deferred +from twisted.trial import unittest + +from scrapy import Spider, signals, Request +from scrapy.utils.test import get_crawler + +from tests.mockserver import MockServer + + +class SimplePipeline: + def process_item(self, item, spider): + item['pipeline_passed'] = True + return item + + +class DeferredPipeline: + def cb(self, item): + item['pipeline_passed'] = True + return item + + def process_item(self, item, spider): + d = Deferred() + d.addCallback(self.cb) + d.callback(item) + return d + + +class ItemSpider(Spider): + name = 'itemspider' + + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + + def parse(self, response): + return {'field': 42} + + +class PipelineTestCase(unittest.TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + def _on_item_scraped(self, item): + self.assertIsInstance(item, dict) + self.assertTrue(item.get('pipeline_passed')) + self.items.append(item) + + def _create_crawler(self, pipeline_class): + settings = { + 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1}, + } + crawler = get_crawler(ItemSpider, settings) + crawler.signals.connect(self._on_item_scraped, signals.item_scraped) + self.items = [] + return crawler + + @defer.inlineCallbacks + def test_simple_pipeline(self): + crawler = self._create_crawler(SimplePipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) + + @defer.inlineCallbacks + def test_deferred_pipeline(self): + crawler = self._create_crawler(DeferredPipeline) + yield crawler.crawl(mockserver=self.mockserver) + self.assertEqual(len(self.items), 1) diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index ae1236bcb..188ec68dd 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -1,38 +1,53 @@ import json import os -import time +import re +import sys +from subprocess import Popen, PIPE +from urllib.parse import urlsplit, urlunsplit -from six.moves.urllib.parse import urlsplit, urlunsplit -from threading import Thread -from libmproxy import controller, proxy -from netlib import http_auth +import pytest from testfixtures import LogCapture - from twisted.internet import defer from twisted.trial.unittest import TestCase -from scrapy.utils.test import get_crawler + from scrapy.http import Request -from tests.spiders import SimpleSpider, SingleRequestSpider +from scrapy.utils.test import get_crawler + from tests.mockserver import MockServer +from tests.spiders import SimpleSpider, SingleRequestSpider -class HTTPSProxy(controller.Master, Thread): +class MitmProxy: + auth_user = 'scrapy' + auth_pass = 'scrapy' - def __init__(self): - password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') - authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + def start(self): + from scrapy.utils.test import get_testenv + script = """ +import sys +from mitmproxy.tools.main import mitmdump +sys.argv[0] = "mitmdump" +sys.exit(mitmdump()) + """ cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'keys', 'mitmproxy-ca.pem') - server = proxy.ProxyServer(proxy.ProxyConfig( - authenticator = authenticator, - cacert = cert_path), - 0) - self.server = server - Thread.__init__(self) - controller.Master.__init__(self, server) + 'keys', 'mitmproxy-ca.pem') + self.proc = Popen([sys.executable, + '-c', script, + '--listen-host', '127.0.0.1', + '--listen-port', '0', + '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass), + '--certs', cert_path, + '--ssl-insecure', + ], + stdout=PIPE, env=get_testenv()) + line = self.proc.stdout.readline().decode('utf-8') + host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1) + address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port) + return address - def http_address(self): - return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname() + def stop(self): + self.proc.kill() + self.proc.communicate() def _wrong_credentials(proxy_url): @@ -40,6 +55,7 @@ def _wrong_credentials(proxy_url): bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@') return urlunsplit(bad_auth_proxy) + class ProxyConnectTestCase(TestCase): def setUp(self): @@ -47,17 +63,14 @@ class ProxyConnectTestCase(TestCase): self.mockserver.__enter__() self._oldenv = os.environ.copy() - self._proxy = HTTPSProxy() - self._proxy.start() - - # Wait for the proxy to start. - time.sleep(1.0) - os.environ['https_proxy'] = self._proxy.http_address() - os.environ['http_proxy'] = self._proxy.http_address() + self._proxy = MitmProxy() + proxy_url = self._proxy.start() + os.environ['https_proxy'] = proxy_url + os.environ['http_proxy'] = proxy_url def tearDown(self): self.mockserver.__exit__(None, None, None) - self._proxy.shutdown() + self._proxy.stop() os.environ = self._oldenv @defer.inlineCallbacks @@ -67,15 +80,7 @@ class ProxyConnectTestCase(TestCase): yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) self._assert_got_response_code(200, l) - @defer.inlineCallbacks - def test_https_noconnect(self): - proxy = os.environ['https_proxy'] - os.environ['https_proxy'] = proxy + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(200, l) - + @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6) @defer.inlineCallbacks def test_https_connect_tunnel_error(self): crawler = get_crawler(SimpleSpider) @@ -100,17 +105,9 @@ class ProxyConnectTestCase(TestCase): with LogCapture() as l: yield crawler.crawl(seed=request) self._assert_got_response_code(200, l) - echo = json.loads(crawler.spider.meta['responses'][0].body) + echo = json.loads(crawler.spider.meta['responses'][0].text) self.assertTrue('Proxy-Authorization' not in echo['headers']) - @defer.inlineCallbacks - def test_https_noconnect_auth_error(self): - os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect' - crawler = get_crawler(SimpleSpider) - with LogCapture() as l: - yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True)) - self._assert_got_response_code(407, l) - def _assert_got_response_code(self, code, log): print(log) self.assertEqual(str(log).count('Crawled (%d)' % code), 1) diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py index f89042b3d..d5a3371ab 100644 --- a/tests/test_responsetypes.py +++ b/tests/test_responsetypes.py @@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypes from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers + class ResponseTypesTest(unittest.TestCase): def test_from_filename(self): diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index cd7480e33..27d79437b 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -5,7 +5,7 @@ from twisted.trial import unittest def reppy_available(): # check if reppy parser is installed try: - from reppy.robots import Robots + from reppy.robots import Robots # noqa: F401 except ImportError: return False return True @@ -14,19 +14,21 @@ def reppy_available(): def rerp_available(): # check if robotexclusionrulesparser is installed try: - from robotexclusionrulesparser import RobotExclusionRulesParser + from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401 except ImportError: return False return True + def protego_available(): # check if protego parser is installed try: - from protego import Protego + from protego import Protego # noqa: F401 except ImportError: return False return True + class BaseRobotParserTest: def _setUp(self, parser_cls): self.parser_cls = parser_cls diff --git a/tests/test_selector.py b/tests/test_selector.py index b2565dd78..09c2546fb 100644 --- a/tests/test_selector.py +++ b/tests/test_selector.py @@ -1,9 +1,9 @@ -import warnings import weakref + from twisted.trial import unittest + from scrapy.http import TextResponse, HtmlResponse, XmlResponse from scrapy.selector import Selector -from lxml import etree class SelectorTestCase(unittest.TestCase): diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 32e65bed5..fda44653a 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -1,4 +1,3 @@ -import six import unittest from unittest import mock @@ -10,7 +9,7 @@ from . import default_settings class SettingsGlobalFuncsTest(unittest.TestCase): def test_get_settings_priority(self): - for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES): + for prio_str, prio_num in SETTINGS_PRIORITIES.items(): self.assertEqual(get_settings_priority(prio_str), prio_num) self.assertEqual(get_settings_priority(99), 99) @@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase): new_dict = {'three': 11, 'four': 21} attribute.set(new_dict, 10) self.assertIsInstance(attribute.value, BaseSettings) - six.assertCountEqual(self, attribute.value, new_dict) - six.assertCountEqual(self, original_settings, original_dict) + self.assertCountEqual(attribute.value, new_dict) + self.assertCountEqual(original_settings, original_dict) new_settings = BaseSettings({'five': 12}, 0) attribute.set(new_settings, 0) # Insufficient priority - six.assertCountEqual(self, attribute.value, new_dict) + self.assertCountEqual(attribute.value, new_dict) attribute.set(new_settings, 10) - six.assertCountEqual(self, attribute.value, new_settings) + self.assertCountEqual(attribute.value, new_settings) def test_repr(self): self.assertEqual(repr(self.attribute), @@ -148,10 +147,10 @@ class BaseSettingsTest(unittest.TestCase): self.settings.setmodule( 'tests.test_settings.default_settings', 10) - self.assertCountEqual(six.iterkeys(self.settings.attributes), - six.iterkeys(ctrl_attributes)) + self.assertCountEqual(self.settings.attributes.keys(), + ctrl_attributes.keys()) - for key in six.iterkeys(ctrl_attributes): + for key in ctrl_attributes.keys(): attr = self.settings.attributes[key] ctrl_attr = ctrl_attributes[key] self.assertEqual(attr.value, ctrl_attr.value) @@ -227,7 +226,7 @@ class BaseSettingsTest(unittest.TestCase): } settings = self.settings settings.attributes = {key: SettingsAttribute(value, 0) for key, value - in six.iteritems(test_configuration)} + in test_configuration.items()} self.assertTrue(settings.getbool('TEST_ENABLED1')) self.assertTrue(settings.getbool('TEST_ENABLED2')) @@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase): 'TEST': BaseSettings({1: 10, 3: 30}, 'default'), 'HASNOBASE': BaseSettings({3: 3000}, 'default')}) s['TEST'].set(2, 200, 'cmdline') - six.assertCountEqual(self, s.getwithbase('TEST'), - {1: 1, 2: 200, 3: 30}) - six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE']) + self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30}) + self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE']) self.assertEqual(s.getwithbase('NONEXISTENT'), {}) def test_maxpriority(self): diff --git a/tests/test_spider.py b/tests/test_spider.py index c0fccfdd6..317a27076 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -15,7 +15,6 @@ from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \ CSVFeedSpider, SitemapSpider from scrapy.linkextractors import LinkExtractor from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.trackref import object_ref from scrapy.utils.test import get_crawler @@ -385,6 +384,14 @@ class CrawlSpiderTest(SpiderTest): self.assertTrue(hasattr(spider, '_follow_links')) self.assertFalse(spider._follow_links) + def test_start_url(self): + spider = self.spider_class("example.com") + spider.start_url = 'https://www.example.com' + + with self.assertRaisesRegex(AttributeError, + r'^Crawling could not start.*$'): + list(spider.start_requests()) + class SitemapSpiderTest(SpiderTest): diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index 106da798c..d8be6e277 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self): diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py index 35b71870a..dbd1fb123 100644 --- a/tests/test_spiderloader/test_spiders/nested/spider4.py +++ b/tests/test_spiderloader/test_spiders/nested/spider4.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider4(Spider): name = "spider4" allowed_domains = ['spider4.com'] diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py index 75a90794e..af679dbd6 100644 --- a/tests/test_spiderloader/test_spiders/spider0.py +++ b/tests/test_spiderloader/test_spiders/spider0.py @@ -1,4 +1,5 @@ from scrapy.spiders import Spider + class Spider0(Spider): allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py index 76efddc7f..6b4317a90 100644 --- a/tests/test_spiderloader/test_spiders/spider1.py +++ b/tests/test_spiderloader/test_spiders/spider1.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider1(Spider): name = "spider1" allowed_domains = ["scrapy1.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py index 0badd8437..352601863 100644 --- a/tests/test_spiderloader/test_spiders/spider2.py +++ b/tests/test_spiderloader/test_spiders/spider2.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider2(Spider): name = "spider2" allowed_domains = ["scrapy2.org", "scrapy3.org"] diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py index d406f2d4f..84998ba35 100644 --- a/tests/test_spiderloader/test_spiders/spider3.py +++ b/tests/test_spiderloader/test_spiders/spider3.py @@ -1,5 +1,6 @@ from scrapy.spiders import Spider + class Spider3(Spider): name = "spider3" allowed_domains = ['spider3.com'] diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 7e4af0d4c..992e60be2 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -1,13 +1,12 @@ from unittest import TestCase - -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse +import warnings from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.spidermiddlewares.offsite import OffsiteMiddleware -from scrapy.spidermiddlewares.offsite import URLWarning +from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning from scrapy.utils.test import get_crawler -import warnings + class TestOffsiteMiddleware(TestCase): @@ -53,6 +52,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware): out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) + class TestOffsiteMiddleware3(TestOffsiteMiddleware2): def _get_spider(self): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 6f8727a15..5b7b5e7aa 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -6,7 +6,6 @@ from twisted.internet import defer from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -from tests.spiders import MockServerSpider class LogExceptionMiddleware: @@ -34,6 +33,7 @@ class RecoverySpider(Spider): if not response.meta.get('dont_fail'): raise TabError() + class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) @@ -50,6 +50,7 @@ class FailProcessSpiderInputMiddleware: spider.logger.info('Middleware: will raise IndexError') raise IndexError() + class ProcessSpiderInputSpiderWithoutErrback(Spider): name = 'ProcessSpiderInputSpiderWithoutErrback' custom_settings = { @@ -177,6 +178,7 @@ class GeneratorRecoverMiddleware: spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} + class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass @@ -247,6 +249,7 @@ class NotGeneratorRecoverMiddleware: spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] + class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): pass diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 21439c20e..7cc17600c 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -1,8 +1,7 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from unittest import TestCase import warnings -from scrapy.exceptions import NotConfigured from scrapy.http import Response, Request from scrapy.settings import Settings from scrapy.spiders import Spider @@ -349,6 +348,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware): ] + # --- Tests using Request meta dict to set policy class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware): req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT} @@ -518,14 +518,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} + class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} + class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} + class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): """ The empty string means "no-referrer-when-downgrade" @@ -545,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware): (301, 'http://scrapytest.org/3'), (301, 'http://scrapytest.org/4'), ), - b'http://scrapytest.org/1', # expected initial referer - b'http://scrapytest.org/1', # expected referer for the redirection request + b'http://scrapytest.org/1', # expected initial referer + b'http://scrapytest.org/1', # expected referer for the redirection request ), ( 'https://scrapytest.org/1', 'https://scrapytest.org/2', @@ -606,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect): (301, 'http://scrapytest.org/3'), (301, 'http://scrapytest.org/4'), ), - None, # expected initial "Referer" - None, # expected "Referer" for the redirection request + None, # expected initial "Referer" + None, # expected "Referer" for the redirection request ), ( 'https://scrapytest.org/1', 'https://scrapytest.org/2', @@ -645,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial "Referer" - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial "Referer" + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', @@ -754,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', @@ -824,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect): (301, 'http://scrapytest.org/103'), (301, 'http://scrapytest.org/104'), ), - b'http://scrapytest.org/101', # expected initial referer - b'http://scrapytest.org/101', # expected referer for the redirection request + b'http://scrapytest.org/101', # expected initial referer + b'http://scrapytest.org/101', # expected referer for the redirection request ), ( 'https://scrapytest.org/201', 'https://scrapytest.org/202', diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 3ded5c027..d5fcf2f7f 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -7,16 +7,20 @@ from scrapy.http import Request from scrapy.loader import ItemLoader from scrapy.selector import Selector + class TestItem(Item): name = Field() + def _test_procesor(x): return x + x + class TestLoader(ItemLoader): default_item_class = TestItem name_out = staticmethod(_test_procesor) + def nonserializable_object_test(self): q = self.queue() try: @@ -35,6 +39,7 @@ def nonserializable_object_test(self): sel = Selector(text='<html><body><p>some text</p></body></html>') self.assertRaises(ValueError, q.push, sel) + class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): chunksize = 100000 @@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): test_nonserializable_object = nonserializable_object_test + class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 1 + class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 2 + class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 3 + class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest): chunksize = 4 @@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest): self.assertEqual(r.url, r2.url) assert r2.meta['request'] is r2 + class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 1 + class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 2 + class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 3 + class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest): chunksize = 4 diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py index 91bbe43bc..fdc5df166 100644 --- a/tests/test_toplevel.py +++ b/tests/test_toplevel.py @@ -1,12 +1,12 @@ from unittest import TestCase -import six + import scrapy class ToplevelTestCase(TestCase): def test_version(self): - self.assertIs(type(scrapy.__version__), six.text_type) + self.assertIs(type(scrapy.__version__), str) def test_version_info(self): self.assertIs(type(scrapy.version_info), tuple) diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py index 22e39821c..bea0cf3e5 100644 --- a/tests/test_urlparse_monkeypatches.py +++ b/tests/test_urlparse_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse import unittest diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index c2211848c..380c41367 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -14,6 +14,7 @@ try: except ImportError: ipy = False + class UtilsConsoleTestCase(unittest.TestCase): def test_get_shell_embed_func(self): diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index c5655df7e..50e1bfd5f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -1,7 +1,6 @@ import unittest import warnings -from six import assertRaisesRegex from w3lib.http import basic_auth_header from scrapy import Request @@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) def test_too_few_arguments_error(self): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, r"too few arguments|the following arguments are required:\s*url", lambda: curl_to_request_kwargs("curl"), @@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase): self.assertEqual(curl_to_request_kwargs(curl_command), expected_result) # case 2: ignore_unknown_options=False (raise exception): - assertRaisesRegex( - self, + self.assertRaisesRegex( ValueError, "Unrecognized options:.*--bar.*--baz", lambda: curl_to_request_kwargs( diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 53228fc6e..38a25778e 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase): self.assertIn(20, d) self.assertNotIn(15, d) - def test_six_range(self): - import six.moves - seq = six.moves.range(10**3, 10**6) - d = SequenceExclude(seq) - self.assertIn(10**2, d) - self.assertIn(10**7, d) - self.assertNotIn(10**4, d) - def test_range_step(self): seq = range(10, 20, 3) d = SequenceExclude(seq) diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 003bb9b02..dfbe71ae2 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -5,8 +5,6 @@ from twisted.python.failure import Failure from scrapy.utils.defer import mustbe_deferred, process_chain, \ process_chain_both, process_parallel, iter_errback -from six.moves import xrange - class MustbeDeferredTest(unittest.TestCase): def test_success_function(self): @@ -16,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase): return steps dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + steps.append(2) # add another value, that should be catched by assertEqual return dfd def test_unfired_deferred(self): @@ -29,18 +27,27 @@ class MustbeDeferredTest(unittest.TestCase): return dfd dfd = mustbe_deferred(_append, 1) - dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred - steps.append(2) # add another value, that should be catched by assertEqual + dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred + steps.append(2) # add another value, that should be catched by assertEqual return dfd + def cb1(value, arg1, arg2): return "(cb1 %s %s %s)" % (value, arg1, arg2) + + def cb2(value, arg1, arg2): return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2)) + + def cb3(value, arg1, arg2): return "(cb3 %s %s %s)" % (value, arg1, arg2) + + def cb_fail(value, arg1, arg2): return Failure(TypeError()) + + def eb1(failure, arg1, arg2): return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2) @@ -83,7 +90,7 @@ class IterErrbackTest(unittest.TestCase): def test_iter_errback_good(self): def itergood(): - for x in xrange(10): + for x in range(10): yield x errors = [] @@ -93,7 +100,7 @@ class IterErrbackTest(unittest.TestCase): def test_iter_errback_bad(self): def iterbad(): - for x in xrange(10): + for x in range(10): if x == 5: a = 1/0 yield x diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index ce04e7f29..159ef8f25 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import import inspect import unittest from unittest import mock diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py index 2524153ea..2fac3da1f 100644 --- a/tests/test_utils_http.py +++ b/tests/test_utils_http.py @@ -2,6 +2,7 @@ import unittest from scrapy.utils.http import decode_chunked_transfer + class ChunkedTest(unittest.TestCase): def test_decode_chunked_transfer(self): @@ -12,7 +13,7 @@ class ChunkedTest(unittest.TestCase): chunked_body += "8\r\n" + "sequence\r\n" chunked_body += "0\r\n\r\n" body = decode_chunked_transfer(chunked_body) - self.assertEqual(body, \ - "This is the data in the first chunk\r\n" + - "and this is the second one\r\n" + - "consequence") + self.assertEqual(body, + "This is the data in the first chunk\r\n" + + "and this is the second one\r\n" + + "consequence") diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py index 4f9f7a370..cf8ad1f23 100644 --- a/tests/test_utils_httpobj.py +++ b/tests/test_utils_httpobj.py @@ -1,9 +1,10 @@ import unittest -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached + class HttpobjUtilsTest(unittest.TestCase): def test_urlparse_cached(self): diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 2d845697e..9776dfb2a 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- import os -import six + from twisted.trial import unittest from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from tests import get_testdata + FOOBAR_NL = u"foo\nbar" @@ -235,6 +236,7 @@ class LxmlXmliterTestCase(XmliterTestCase): i = self.xmliter(42, 'product') self.assertRaises(TypeError, next, i) + class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv') @@ -255,8 +257,8 @@ class UtilsCsvTestCase(unittest.TestCase): # explicit type check cuz' we no like stinkin' autocasting! yarrr for result_row in result: - self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys()))) - self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values()))) + self.assertTrue(all((isinstance(k, str) for k in result_row.keys()))) + self.assertTrue(all((isinstance(v, str) for v in result_row.values()))) def test_csviter_delimiter(self): body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t') diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 742e04803..2c23f3616 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import print_function import sys import logging import unittest diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index a94398796..b79e0ac1c 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,16 +1,17 @@ -import gc import functools +import gc import operator +import platform import unittest from itertools import count -import platform -import six +from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) + __doctests__ = ['scrapy.utils.python'] @@ -21,8 +22,12 @@ class MutableChainTest(unittest.TestCase): m.extend([7, 8]) m.extend([9, 10], (11, 12)) self.assertEqual(next(m), 0) - self.assertEqual(m.next(), 1) - self.assertEqual(m.__next__(), 2) + self.assertEqual(m.__next__(), 1) + with catch_warnings(record=True) as warnings: + self.assertEqual(m.next(), 2) + self.assertEqual(len(warnings), 1) + self.assertIn('scrapy.utils.python.MutableChain.__next__', + str(warnings[0].message)) self.assertEqual(list(m), list(range(3, 13))) @@ -163,33 +168,6 @@ class UtilsPythonTestCase(unittest.TestCase): gc.collect() self.assertFalse(len(wk._weakdict)) - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict(self): - d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'} - d2 = stringify_dict(d, keys_only=False) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) - - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict_tuples(self): - tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')] - d = dict(tuples) - d2 = stringify_dict(tuples, keys_only=False) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys()) - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values())) - - @unittest.skipUnless(six.PY2, "deprecated function") - def test_stringify_dict_keys_only(self): - d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'} - d2 = stringify_dict(d) - self.assertEqual(d, d2) - self.assertIsNot(d, d2) # shouldn't modify in place - self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys())) - def test_get_func_args(self): def f1(a, b, c): pass @@ -227,12 +205,12 @@ class UtilsPythonTestCase(unittest.TestCase): if platform.python_implementation() == 'CPython': # TODO: how do we fix this to return the actual argument names? - self.assertEqual(get_func_args(six.text_type.split), []) + self.assertEqual(get_func_args(str.split), []) self.assertEqual(get_func_args(" ".join), []) self.assertEqual(get_func_args(operator.itemgetter(2)), []) else: self.assertEqual( - get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit']) + get_func_args(str.split, stripself=True), ['sep', 'maxsplit']) self.assertEqual(get_func_args(" ".join, stripself=True), ['list']) self.assertEqual( get_func_args(operator.itemgetter(2), stripself=True), ['obj']) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 92cd16de7..06d9c004c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,8 +1,4 @@ -# -*- coding: utf-8 -*- import unittest -import sys - -import six from scrapy.http import Request, FormRequest from scrapy.spiders import Spider diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index 625a32048..3e664fc74 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -1,9 +1,9 @@ -from __future__ import print_function import unittest from scrapy.http import Request from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \ request_authenticate, request_httprepr + class UtilsRequestTest(unittest.TestCase): def test_request_fingerprint(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index bea4dade3..6ebf290c0 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,12 +1,13 @@ import os import unittest -from six.moves.urllib.parse import urlparse +from urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import (response_httprepr, open_in_browser, get_meta_refresh, get_base_url, response_status_message) + __doctests__ = ['scrapy.utils.response'] diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 62edd420d..16b7c5c68 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest): def _get_result(self, signal, *a, **kw): return send_catch_log_deferred(signal, *a, **kw) + class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 716bb44eb..db323ab31 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -2,6 +2,7 @@ import unittest from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots + class SitemapTest(unittest.TestCase): def test_sitemap(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index d9de1ce77..ee7d17062 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,20 +1,19 @@ import unittest + +from scrapy import Spider from scrapy.http import Request from scrapy.item import BaseItem from scrapy.utils.spider import iterate_spider_output, iter_spider_classes -from scrapy.spiders import CrawlSpider - -class MyBaseSpider(CrawlSpider): - pass # abstract spider - -class MySpider1(MyBaseSpider): +class MySpider1(Spider): name = 'myspider1' -class MySpider2(MyBaseSpider): + +class MySpider2(Spider): name = 'myspider2' + class UtilsSpidersTestCase(unittest.TestCase): def test_iterate_spider_output(self): @@ -32,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase): it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 480a717e7..16e02f919 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -1,6 +1,7 @@ -import six import unittest +from io import StringIO from unittest import mock + from scrapy.utils import trackref @@ -38,12 +39,12 @@ Live References Bar 1 oldest: 0s ago ''') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_empty(self, stdout): trackref.print_live_refs() self.assertEqual(stdout.getvalue(), 'Live References\n\n\n') - @mock.patch('sys.stdout', new_callable=six.StringIO) + @mock.patch('sys.stdout', new_callable=StringIO) def test_print_live_refs_with_objects(self, stdout): o1 = Foo() # NOQA trackref.print_live_refs() diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index e6588055c..21e9a056a 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- import unittest -import six -from six.moves.urllib.parse import urlparse - from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - add_http_if_no_scheme, guess_scheme, - parse_url, strip_url) + add_http_if_no_scheme, guess_scheme, strip_url) + __doctests__ = ['scrapy.utils.url'] @@ -187,6 +184,7 @@ class AddHttpIfNoScheme(unittest.TestCase): class GuessSchemeTest(unittest.TestCase): pass + def create_guess_scheme_t(args): def do_expected(self): url = guess_scheme(args[0]) @@ -195,6 +193,7 @@ def create_guess_scheme_t(args): args[0], url, args[1]) return do_expected + def create_skipped_scheme_t(args): def do_expected(self): raise unittest.SkipTest(args[2]) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index a81946490..746367b41 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -3,7 +3,6 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ import os -import six import shutil import OpenSSL.SSL @@ -78,26 +77,6 @@ class ParseUrlTestCase(unittest.TestCase): to_bytes(x) if not isinstance(x, int) else x for x in test) self.assertEqual(client._parse(url), test, url) - def test_externalUnicodeInterference(self): - """ - L{client._parse} should return C{str} for the scheme, host, and path - elements of its return tuple, even when passed an URL which has - previously been passed to L{urlparse} as a C{unicode} string. - """ - if not six.PY2: - raise unittest.SkipTest( - "Applies only to Py2, as urls can be ONLY unicode on Py3") - badInput = u'http://example.com/path' - goodInput = badInput.encode('ascii') - self._parse(badInput) # cache badInput in urlparse_cached - scheme, netloc, host, port, path = self._parse(goodInput) - self.assertTrue(isinstance(scheme, str)) - self.assertTrue(isinstance(netloc, str)) - self.assertTrue(isinstance(host, str)) - self.assertTrue(isinstance(path, str)) - self.assertTrue(isinstance(port, int)) - - class ScrapyHTTPPageGetterTests(unittest.TestCase): @@ -318,7 +297,7 @@ class WebClientTestCase(unittest.TestCase): def cleanup(passthrough): # Clean up the server which is hanging around not doing # anything. - connected = list(six.iterkeys(self.wrapper.protocols)) + connected = list(self.wrapper.protocols.keys()) # There might be nothing here if the server managed to already see # that the connection was lost. if connected: diff --git a/tox.ini b/tox.ini index 195cc106a..1ec8f52e4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,15 +4,12 @@ # and then run "tox" from this directory. [tox] -envlist = py35 - -[latest] -basepython = python3.8 +envlist = security,flake8,py3 +minversion = 1.7.0 [testenv] deps = -ctests/constraints.txt - -rrequirements-py3.txt -rtests/requirements-py3.txt # Extras botocore>=1.3.23 @@ -26,11 +23,28 @@ passenv = commands = py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests} -[testenv:py35] -basepython = python3.5 +[testenv:security] +basepython = python3 +deps = + bandit +commands = + bandit -r -c .bandit.yml {posargs:scrapy} -[testenv:py35-pinned] -basepython = python3.5 +[testenv:flake8] +basepython = python3 +deps = + {[testenv]deps} + pytest-flake8 +commands = + py.test --flake8 {posargs:docs scrapy tests} + +[testenv:pypy3] +basepython = pypy3 +commands = + py.test {posargs:--durations=10 docs scrapy tests} + +[testenv:pinned] +basepython = python3 deps = -ctests/constraints.txt cryptography==2.0 @@ -51,34 +65,11 @@ deps = botocore==1.3.23 Pillow==3.4.2 -[testenv:py36] -basepython = python3.6 - -[testenv:py37] -basepython = python3.7 - -[testenv:py38] -basepython = python3.8 - -[testenv:pypy3] -basepython = pypy3 -commands = - py.test {posargs:--durations=10 docs scrapy tests} - -[testenv:security] -basepython = {[latest]basepython} -deps = - bandit -commands = - bandit -r -c .bandit.yml {posargs:scrapy} - -[testenv:flake8] -basepython = {[latest]basepython} +[testenv:extra-deps] deps = {[testenv]deps} - pytest-flake8 -commands = - py.test --flake8 {posargs:docs scrapy tests} + reppy + robotexclusionrulesparser [docs] changedir = docs @@ -86,29 +77,22 @@ deps = -rdocs/requirements.txt [testenv:docs] -basepython = {[latest]basepython} +basepython = python3 changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -W -b html . {envtmpdir}/html [testenv:docs-coverage] -basepython = {[latest]basepython} +basepython = python3 changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -b coverage . {envtmpdir}/coverage [testenv:docs-links] -basepython = {[latest]basepython} +basepython = python3 changedir = {[docs]changedir} deps = {[docs]deps} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck - -[testenv:py38-extra-deps] -basepython = python3.8 -deps = - {[testenv]deps} - reppy - robotexclusionrulesparser