diff --git a/.bandit.yml b/.bandit.yml
new file mode 100644
index 000000000..243379b0b
--- /dev/null
+++ b/.bandit.yml
@@ -0,0 +1,18 @@
+skips:
+- B101
+- B105
+- B301
+- B303
+- B306
+- B307
+- B311
+- B320
+- B321
+- B402 # https://github.com/scrapy/scrapy/issues/4180
+- B403
+- B404
+- B406
+- B410
+- B503
+- B603
+- B605
diff --git a/.travis.yml b/.travis.yml
index 9840e3c20..b7aa0612a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,15 +7,16 @@ branches:
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
+ - env: TOXENV=security
+ python: 3.8
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pylint
python: 3.8
- env: TOXENV=pypy3
- python: 3.5
- env: TOXENV=py35
python: 3.5
- - env: TOXENV=py35-pinned
+ - env: TOXENV=pinned
python: 3.5
- env: TOXENV=py36
python: 3.6
@@ -23,7 +24,7 @@ matrix:
python: 3.7
- env: TOXENV=py38
python: 3.8
- - env: TOXENV=py38-extra-deps
+ - env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=docs
python: 3.6
diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md
index d477168eb..d1cd3e517 100644
--- a/CODE_OF_CONDUCT.md
+++ b/CODE_OF_CONDUCT.md
@@ -68,7 +68,7 @@ members of the project's leadership.
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at [http://contributor-covenant.org/version/1/4][version]
+available at [http://contributor-covenant.org/version/1/4][version].
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
diff --git a/README.rst b/README.rst
index fb4ca8e4f..7fefaeec9 100644
--- a/README.rst
+++ b/README.rst
@@ -34,8 +34,8 @@ Scrapy is a fast high-level web crawling and web scraping framework, used to
crawl websites and extract structured data from their pages. It can be used for
a wide range of purposes, from data mining to monitoring and automated testing.
-For more information including a list of features check the Scrapy homepage at:
-https://scrapy.org
+Check the Scrapy homepage at https://scrapy.org for more information,
+including a list of features.
Requirements
============
@@ -50,8 +50,8 @@ The quick way::
pip install scrapy
-For more details see the install section in the documentation:
-https://docs.scrapy.org/en/latest/intro/install.html
+See the install section in the documentation at
+https://docs.scrapy.org/en/latest/intro/install.html for more details.
Documentation
=============
@@ -62,17 +62,17 @@ directory.
Releases
========
-You can find release notes at https://docs.scrapy.org/en/latest/news.html
+You can check https://docs.scrapy.org/en/latest/news.html for the release notes.
Community (blog, twitter, mail list, IRC)
=========================================
-See https://scrapy.org/community/
+See https://scrapy.org/community/ for details.
Contributing
============
-See https://docs.scrapy.org/en/master/contributing.html
+See https://docs.scrapy.org/en/master/contributing.html for details.
Code of Conduct
---------------
@@ -86,9 +86,9 @@ Please report unacceptable behavior to opensource@scrapinghub.com.
Companies using Scrapy
======================
-See https://scrapy.org/companies/
+See https://scrapy.org/companies/ for a list.
Commercial Support
==================
-See https://scrapy.org/support/
+See https://scrapy.org/support/ for details.
diff --git a/conftest.py b/conftest.py
index d5d61ddd3..d37c22436 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,18 +1,23 @@
-import six
+from pathlib import Path
+
import pytest
+def _py_files(folder):
+ return (str(p) for p in Path(folder).rglob('*.py'))
+
+
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
+ # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+ *_py_files("tests/CrawlerProcess")
]
-
-if six.PY3:
- for line in open('tests/py3-ignores.txt'):
- file_path = line.strip()
- if file_path and file_path[0] != '#':
- collect_ignore.append(file_path)
+for line in open('tests/ignores.txt'):
+ file_path = line.strip()
+ if file_path and file_path[0] != '#':
+ collect_ignore.append(file_path)
@pytest.fixture()
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html
new file mode 100644
index 000000000..71aff8847
--- /dev/null
+++ b/docs/_tests/quotes1.html
@@ -0,0 +1,281 @@
+
+
+
+ Quotes to Scrape
+
+
+
+
+
+
+
+
+
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is our choices, Harry, that show what we truly are, far more than our abilities.”
+
by J.K. Rowling
+ (about)
+
+
+
+
+
+
“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”
+
by Jane Austen
+ (about)
+
+
+
+
+
+
“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”
+
by Marilyn Monroe
+ (about)
+
+
+
+
+
+
“Try not to become a man of success. Rather become a man of value.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is better to be hated for what you are than to be loved for what you are not.”
+
by André Gide
+ (about)
+
+
+
+
+
+
“I have not failed. I've just found 10,000 ways that won't work.”
+
by Thomas A. Edison
+ (about)
+
+
+
+
+
+
“A woman is like a tea bag; you never know how strong it is until it's in hot water.”
+
by Eleanor Roosevelt
+ (about)
+
+
+
+
+
+
“A day without sunshine is like, you know, night.”
+
by Steve Martin
+ (about)
+
+
+
+
+
+
+
+
+
+
+
+
`. Running tests requires
+:doc:`tox `.
.. _running-tests:
Running tests
-------------
-Make sure you have a recent enough `tox`_ installation:
+To run all tests::
- ``tox --version``
-
-If your version is older than 1.7.0, please update it first:
-
- ``pip install -U tox``
-
-To run all tests go to the root directory of Scrapy source code and run:
-
- ``tox``
+ tox
To run a specific test (say ``tests/test_loader.py``) use:
``tox -- tests/test_loader.py``
-To run the tests on a specific tox_ environment, use ``-e `` with an
-environment name from ``tox.ini``. For example, to run the tests with Python
-3.6 use::
+To run the tests on a specific :doc:`tox ` environment, use
+``-e `` with an environment name from ``tox.ini``. For example, to run
+the tests with Python 3.6 use::
tox -e py36
-You can also specify a comma-separated list of environmets, and use `tox’s
-parallel mode`_ to run the tests on multiple environments in parallel::
+You can also specify a comma-separated list of environmets, and use :ref:`tox’s
+parallel mode ` to run the tests on multiple environments in
+parallel::
- tox -e py27,py36 -p auto
+ tox -e py36,py38 -p auto
-To pass command-line options to pytest_, add them after ``--`` in your call to
-tox_. Using ``--`` overrides the default positional arguments defined in
-``tox.ini``, so you must include those default positional arguments
-(``scrapy tests``) after ``--`` as well::
+To pass command-line options to :doc:`pytest `, add them after
+``--`` in your call to :doc:`tox `. Using ``--`` overrides the
+default positional arguments defined in ``tox.ini``, so you must include those
+default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 tox_ environment using all your CPU cores::
+the Python 3.6 :doc:`tox ` environment using all your CPU cores::
tox -e py36 -- scrapy tests -n auto
-To see coverage report install `coverage`_ (``pip install coverage``) and run:
+To see coverage report install :doc:`coverage `
+(``pip install coverage``) and run:
``coverage report``
see output of ``coverage --help`` for more options like html or xml report.
-.. _coverage: https://pypi.python.org/pypi/coverage
-
Writing tests
-------------
@@ -270,13 +262,9 @@ And their unit-tests are in::
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
.. _Scrapy subreddit: https://reddit.com/r/scrapy
-.. _Twisted unit-testing framework: https://twistedmatrix.com/documents/current/core/development/policy/test-standard.html
.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
.. _open issues: https://github.com/scrapy/scrapy/issues
.. _PEP 257: https://www.python.org/dev/peps/pep-0257/
.. _pull request: https://help.github.com/en/articles/creating-a-pull-request
-.. _pytest: https://docs.pytest.org/en/latest/usage.html
-.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html
-.. _tox: https://pypi.python.org/pypi/tox
-.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode
+.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 8b2fef065..01986b594 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').get(),
'author': quote.xpath('span/small/text()').get(),
+ 'text': quote.css('span.text::text').get(),
}
next_page = response.css('li.next a::attr("href")').get()
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 0629b9e19..6b15a5fbd 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -235,13 +235,16 @@ You will see something like::
[s] shelp() Shell help (print this help)
[s] fetch(req_or_url) Fetch request (or URL) and update local objects
[s] view(response) View response in a browser
- >>>
Using the shell, you can try selecting elements using `CSS`_ with the response
-object::
+object:
- >>> response.css('title')
- []
+.. invisible-code-block: python
+
+ response = load_response('http://quotes.toscrape.com/page/1/', 'quotes1.html')
+
+>>> response.css('title')
+[]
The result of running ``response.css('title')`` is a list-like object called
:class:`~scrapy.selector.SelectorList`, which represents a list of
@@ -372,6 +375,9 @@ we want::
We get a list of selectors for the quote HTML elements with::
>>> response.css("div.quote")
+ [,
+ ,
+ ...]
Each of the selectors returned by the query above allows us to run further
queries over their sub-elements. Let's assign the first selector to a
@@ -396,6 +402,12 @@ to get all of them::
>>> tags
['change', 'deep-thoughts', 'thinking', 'world']
+.. invisible-code-block: python
+
+ from sys import version_info
+
+.. skip: next if(version_info < (3, 6), reason="Only Python 3.6+ dictionaries match the output")
+
Having figured out how to extract each bit, we can now iterate over all the
quotes elements and put them together into a Python dictionary::
@@ -404,10 +416,9 @@ quotes elements and put them together into a Python dictionary::
... author = quote.css("small.author::text").get()
... tags = quote.css("div.tags a.tag::text").getall()
... print(dict(text=text, author=author, tags=tags))
- {'tags': ['change', 'deep-thoughts', 'thinking', 'world'], 'author': 'Albert Einstein', 'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”'}
- {'tags': ['abilities', 'choices'], 'author': 'J.K. Rowling', 'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”'}
- ... a few more of these, omitted for brevity
- >>>
+ {'text': '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”', 'author': 'Albert Einstein', 'tags': ['change', 'deep-thoughts', 'thinking', 'world']}
+ {'text': '“It is our choices, Harry, that show what we truly are, far more than our abilities.”', 'author': 'J.K. Rowling', 'tags': ['abilities', 'choices']}
+ ...
Extracting data in our spider
-----------------------------
@@ -521,7 +532,7 @@ There is also an ``attrib`` property available
(see :ref:`selecting-attributes` for more)::
>>> response.css('li.next a').attrib['href']
- '/page/2'
+ '/page/2/'
Let's see now our spider modified to recursively follow the link to the next
page, extracting data from it::
diff --git a/docs/news.rst b/docs/news.rst
index 669844045..9dfd28508 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -308,12 +308,12 @@ New features
convenient way to build JSON requests (:issue:`3504`, :issue:`3505`)
* A ``process_request`` callback passed to the :class:`~scrapy.spiders.Rule`
- constructor now receives the :class:`~scrapy.http.Response` object that
+ ``__init__`` method now receives the :class:`~scrapy.http.Response` object that
originated the request as its second argument (:issue:`3682`)
* A new ``restrict_text`` parameter for the
:attr:`LinkExtractor `
- constructor allows filtering links by linking text (:issue:`3622`,
+ ``__init__`` method allows filtering links by linking text (:issue:`3622`,
:issue:`3635`)
* A new :setting:`FEED_STORAGE_S3_ACL` setting allows defining a custom ACL
@@ -479,7 +479,7 @@ The following deprecated APIs have been removed (:issue:`3578`):
* From :class:`~scrapy.selector.Selector`:
- * ``_root`` (both the constructor argument and the object property, use
+ * ``_root`` (both the ``__init__`` method argument and the object property, use
``root``)
* ``extract_unquoted`` (use ``getall``)
@@ -2703,7 +2703,7 @@ Scrapy changes:
- removed ``ENCODING_ALIASES`` setting, as encoding auto-detection has been moved to the `w3lib`_ library
- promoted :ref:`topics-djangoitem` to main contrib
- LogFormatter method now return dicts(instead of strings) to support lazy formatting (:issue:`164`, :commit:`dcef7b0`)
-- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the constructor
+- downloader handlers (:setting:`DOWNLOAD_HANDLERS` setting) now receive settings as the first argument of the ``__init__`` method
- replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
- removed signal: ``scrapy.mail.mail_sent``
- removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled
@@ -2917,7 +2917,7 @@ API changes
- ``Request.copy()`` and ``Request.replace()`` now also copies their ``callback`` and ``errback`` attributes (#231)
- Removed ``UrlFilterMiddleware`` from ``scrapy.contrib`` (already disabled by default)
- Offsite middelware doesn't filter out any request coming from a spider that doesn't have a allowed_domains attribute (#225)
-- Removed Spider Manager ``load()`` method. Now spiders are loaded in the constructor itself.
+- Removed Spider Manager ``load()`` method. Now spiders are loaded in the ``__init__`` method itself.
- Changes to Scrapy Manager (now called "Crawler"):
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 379da9994..773b92cea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,2 +1,4 @@
Sphinx>=2.1
-sphinx_rtd_theme
\ No newline at end of file
+sphinx-hoverxref
+sphinx-notfound-page
+sphinx_rtd_theme
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 7c8c40b5f..1c461a511 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -273,5 +273,3 @@ class (which they all inherit from).
Close the given spider. After this is called, no more specific stats
can be accessed or collected.
-
-.. _reactor: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index 2effe94dc..ae25dfa2f 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -166,11 +166,10 @@ for concurrency.
For more information about asynchronous programming and Twisted see these
links:
-* `Introduction to Deferreds in Twisted`_
+* :doc:`twisted:core/howto/defer-intro`
* `Twisted - hello, asynchronous programming`_
* `Twisted Introduction - Krondo`_
.. _Twisted: https://twistedmatrix.com/trac/
-.. _Introduction to Deferreds in Twisted: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
.. _Twisted - hello, asynchronous programming: http://jessenoller.com/blog/2009/02/11/twisted-hello-asynchronous-programming/
.. _Twisted Introduction - Krondo: http://krondo.com/an-introduction-to-asynchronous-programming-and-twisted/
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index a93bee06b..5b3cd7e75 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -1,3 +1,5 @@
+.. highlight:: none
+
.. _topics-commands:
=================
@@ -66,7 +68,9 @@ structure by default, similar to this::
The directory where the ``scrapy.cfg`` file resides is known as the *project
root directory*. That file contains the name of the python module that defines
-the project settings. Here is an example::
+the project settings. Here is an example:
+
+.. code-block:: ini
[settings]
default = myproject.settings
@@ -80,7 +84,9 @@ A project root directory, the one that contains the ``scrapy.cfg``, may be
shared by multiple Scrapy projects, each with its own settings module.
In that case, you must define one or more aliases for those settings modules
-under ``[settings]`` in your ``scrapy.cfg`` file::
+under ``[settings]`` in your ``scrapy.cfg`` file:
+
+.. code-block:: ini
[settings]
default = myproject1.settings
@@ -277,6 +283,8 @@ check
Run contract checks.
+.. skip: start
+
Usage examples::
$ scrapy check -l
@@ -294,6 +302,8 @@ Usage examples::
[FAILED] first_spider:parse
>>> Returned 92 requests, expected 0..4
+.. skip: end
+
.. command:: list
list
@@ -481,6 +491,8 @@ Supported options:
* ``--verbose`` or ``-v``: display information for each depth level
+.. skip: start
+
Usage example::
$ scrapy parse http://www.example.com/ -c parse_item
@@ -495,6 +507,8 @@ Usage example::
# Requests -----------------------------------------------------------------
[]
+.. skip: end
+
.. command:: settings
@@ -573,7 +587,9 @@ Default: ``''`` (empty string)
A module to use for looking up custom Scrapy commands. This is used to add custom
commands for your Scrapy project.
-Example::
+Example:
+
+.. code-block:: python
COMMANDS_MODULE = 'mybot.commands'
@@ -588,7 +604,11 @@ You can also add Scrapy commands from an external library by adding a
``scrapy.commands`` section in the entry points of the library ``setup.py``
file.
-The following example adds ``my_command`` command::
+The following example adds ``my_command`` command:
+
+.. skip: next
+
+.. code-block:: python
from setuptools import setup, find_packages
diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst
index 0aaad0c77..4b2588518 100644
--- a/docs/topics/debug.rst
+++ b/docs/topics/debug.rst
@@ -48,6 +48,10 @@ The most basic way of checking the output of your spider is to use the
of the spider at the method level. It has the advantage of being flexible and
simple to use, but does not allow debugging code inside a method.
+.. highlight:: none
+
+.. skip: start
+
In order to see the item scraped from a specific url::
$ scrapy parse --spider=myspider -c parse_item -d 2
@@ -85,6 +89,8 @@ using::
$ scrapy parse --spider=myspider -d 3 'http://example.com/page1'
+.. skip: end
+
Scrapy Shell
============
@@ -94,6 +100,8 @@ spider, it is of little help to check what happens inside a callback, besides
showing the response received and the output. How to debug the situation when
``parse_details`` sometimes receives no item?
+.. highlight:: python
+
Fortunately, the :command:`shell` is your bread and butter in this case (see
:ref:`topics-shell-inspect-response`)::
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 8048e1c86..ae6d41809 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -474,7 +474,7 @@ DBM storage backend
A DBM_ storage backend is also available for the HTTP cache middleware.
- By default, it uses the anydbm_ module, but you can change it with the
+ By default, it uses the :mod:`dbm`, but you can change it with the
:setting:`HTTPCACHE_DBM_MODULE` setting.
.. _httpcache-storage-custom:
@@ -626,7 +626,7 @@ HTTPCACHE_DBM_MODULE
.. versionadded:: 0.13
-Default: ``'anydbm'``
+Default: ``'dbm'``
The database module to use in the :ref:`DBM storage backend
`. This setting is specific to the DBM backend.
@@ -1202,4 +1202,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
.. _DBM: https://en.wikipedia.org/wiki/Dbm
-.. _anydbm: https://docs.python.org/2/library/anydbm.html
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
index 949cdc638..72bf52227 100644
--- a/docs/topics/email.rst
+++ b/docs/topics/email.rst
@@ -9,19 +9,19 @@ Sending e-mail
Although Python makes sending e-mails relatively easy via the `smtplib`_
library, Scrapy provides its own facility for sending e-mails which is very
-easy to use and it's implemented using `Twisted non-blocking IO`_, to avoid
-interfering with the non-blocking IO of the crawler. It also provides a
-simple API for sending attachments and it's very easy to configure, with a few
-:ref:`settings `.
+easy to use and it's implemented using :doc:`Twisted non-blocking IO
+`, to avoid interfering with the non-blocking
+IO of the crawler. It also provides a simple API for sending attachments and
+it's very easy to configure, with a few :ref:`settings
+`.
.. _smtplib: https://docs.python.org/2/library/smtplib.html
-.. _Twisted non-blocking IO: https://twistedmatrix.com/documents/current/core/howto/defer-intro.html
Quick example
=============
There are two ways to instantiate the mail sender. You can instantiate it using
-the standard constructor::
+the standard ``__init__`` method::
from scrapy.mail import MailSender
mailer = MailSender()
@@ -39,7 +39,8 @@ MailSender class reference
==========================
MailSender is the preferred class to use for sending emails from Scrapy, as it
-uses `Twisted non-blocking IO`_, like the rest of the framework.
+uses :doc:`Twisted non-blocking IO `, like the
+rest of the framework.
.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
@@ -111,7 +112,7 @@ uses `Twisted non-blocking IO`_, like the rest of the framework.
Mail settings
=============
-These settings define the default constructor values of the :class:`MailSender`
+These settings define the default ``__init__`` method values of the :class:`MailSender`
class, and can be used to configure e-mail notifications in your project without
writing any code (for those extensions and code that uses :class:`MailSender`).
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index a698a6a4e..b8d898022 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -87,8 +87,8 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
-If you use :class:`~.Item` you can declare a serializer in the
-:ref:`field metadata `. The serializer must be
+If you use :class:`~.Item` you can declare a serializer in the
+:ref:`field metadata `. The serializer must be
a callable which receives a value and returns its serialized form.
Example::
@@ -144,7 +144,7 @@ BaseItemExporter
defining what fields to export, whether to export empty fields, or which
encoding to use.
- These features can be configured through the constructor arguments which
+ These features can be configured through the ``__init__`` method arguments which
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
@@ -246,8 +246,8 @@ XmlItemExporter
:param item_element: The name of each item element in the exported XML.
:type item_element: str
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@@ -306,9 +306,9 @@ CsvItemExporter
multi-valued fields, if found.
:type include_headers_line: str
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor, and the leftover arguments to the
- `csv.writer`_ constructor, so you can use any ``csv.writer`` constructor
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to the
+ `csv.writer`_ ``__init__`` method, so you can use any ``csv.writer`` ``__init__`` method
argument to customize this exporter.
A typical output of this exporter would be::
@@ -334,8 +334,8 @@ PickleItemExporter
For more information, refer to the `pickle module documentation`_.
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
Pickle isn't a human readable format, so no output examples are provided.
@@ -351,8 +351,8 @@ PprintItemExporter
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
- The additional keyword arguments of this constructor are passed to the
- :class:`BaseItemExporter` constructor.
+ The additional keyword arguments of this ``__init__`` method are passed to the
+ :class:`BaseItemExporter` ``__init__`` method.
A typical output of this exporter would be::
@@ -367,10 +367,10 @@ JsonItemExporter
.. class:: JsonItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing all
- objects as a list of objects. The additional constructor arguments are
- passed to the :class:`BaseItemExporter` constructor, and the leftover
- arguments to the `JSONEncoder`_ constructor, so you can use any
- `JSONEncoder`_ constructor argument to customize this exporter.
+ objects as a list of objects. The additional ``__init__`` method arguments are
+ passed to the :class:`BaseItemExporter` ``__init__`` method, and the leftover
+ arguments to the `JSONEncoder`_ ``__init__`` method, so you can use any
+ `JSONEncoder`_ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
@@ -398,10 +398,10 @@ JsonLinesItemExporter
.. class:: JsonLinesItemExporter(file, \**kwargs)
Exports Items in JSON format to the specified file-like object, writing one
- JSON-encoded item per line. The additional constructor arguments are passed
- to the :class:`BaseItemExporter` constructor, and the leftover arguments to
- the `JSONEncoder`_ constructor, so you can use any `JSONEncoder`_
- constructor argument to customize this exporter.
+ JSON-encoded item per line. The additional ``__init__`` method arguments are passed
+ to the :class:`BaseItemExporter` ``__init__`` method, and the leftover arguments to
+ the `JSONEncoder`_ ``__init__`` method, so you can use any `JSONEncoder`_
+ ``__init__`` method argument to customize this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 72c2290b5..0a7455ec9 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -28,7 +28,7 @@ Loading & activating extensions
Extensions are loaded and activated at startup by instantiating a single
instance of the extension class. Therefore, all the extension initialization
-code must be performed in the class constructor (``__init__`` method).
+code must be performed in the class ``__init__`` method.
To make an extension available, add it to the :setting:`EXTENSIONS` setting in
your Scrapy settings. In :setting:`EXTENSIONS`, each extension is represented
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index fae18200a..cdc4953c2 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -29,7 +29,8 @@ Each item pipeline component is a Python class that must implement the following
This method is called for every item pipeline component. :meth:`process_item`
must either: return a dict with data, return an :class:`~scrapy.item.Item`
- (or any descendant class) object, return a `Twisted Deferred`_ or raise
+ (or any descendant class) object, return a
+ :class:`~twisted.internet.defer.Deferred` or raise
:exc:`~scrapy.exceptions.DropItem` exception. Dropped items are no longer
processed by further pipeline components.
@@ -67,8 +68,6 @@ Additionally, they may also implement the following methods:
:type crawler: :class:`~scrapy.crawler.Crawler` object
-.. _Twisted Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
-
Item pipeline example
=====================
@@ -166,7 +165,8 @@ method and how to clean up the resources properly.::
Take screenshot of item
-----------------------
-This example demonstrates how to return Deferred_ from :meth:`process_item` method.
+This example demonstrates how to return a
+:class:`~twisted.internet.defer.Deferred` from the :meth:`process_item` method.
It uses Splash_ to render screenshot of item url. Pipeline
makes request to locally running instance of Splash_. After request is downloaded
and Deferred callback fires, it saves item to a file and adds filename to an item.
@@ -209,7 +209,6 @@ and Deferred callback fires, it saves item to a file and adds filename to an ite
return item
.. _Splash: https://splash.readthedocs.io/en/stable/
-.. _Deferred: https://twistedmatrix.com/documents/current/core/howto/defer.html
Duplicates filter
-----------------
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index d70e7428b..cdf60208e 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -16,12 +16,12 @@ especially in a larger project with many spiders.
To define common output data format Scrapy provides the :class:`Item` class.
:class:`Item` objects are simple containers used to collect the scraped data.
They provide a `dictionary-like`_ API with a convenient syntax for declaring
-their available fields.
+their available fields.
-Various Scrapy components use extra information provided by Items:
+Various Scrapy components use extra information provided by Items:
exporters look at declared fields to figure out columns to export,
serialization can be customized using Item fields metadata, :mod:`trackref`
-tracks Item instances to help find memory leaks
+tracks Item instances to help find memory leaks
(see :ref:`topics-leaks-trackrefs`), etc.
.. _dictionary-like: https://docs.python.org/2/library/stdtypes.html#dict
@@ -237,7 +237,7 @@ Item objects
Return a new Item optionally initialized from the given argument.
- Items replicate the standard `dict API`_, including its constructor, and
+ Items replicate the standard `dict API`_, including its ``__init__`` method, and
also provide the following additional API members:
.. automethod:: copy
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 0318e37aa..de3f38023 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -26,7 +26,7 @@ Using Item Loaders to populate items
To use an Item Loader, you must first instantiate it. You can either
instantiate it with a dict-like object (e.g. Item or dict) or without one, in
-which case an Item is automatically instantiated in the Item Loader constructor
+which case an Item is automatically instantiated in the Item Loader ``__init__`` method
using the Item class specified in the :attr:`ItemLoader.default_item_class`
attribute.
@@ -142,20 +142,6 @@ accept one (and only one) positional argument, which will be an iterable.
containing the collected values (for that field). The result of the output
processors is the value that will be finally assigned to the item.
-If you want to use a plain function as a processor, make sure it receives
-``self`` as the first argument::
-
- def lowercase_processor(self, values):
- for v in values:
- yield v.lower()
-
- class MyItemLoader(ItemLoader):
- name_in = lowercase_processor
-
-This is because whenever a function is assigned as a class variable, it becomes
-a method and would be passed the instance as the the first argument when being
-called. See `this answer on stackoverflow`_ for more details.
-
The other thing you need to keep in mind is that the values returned by input
processors are collected internally (in lists) and then passed to output
processors to populate the fields.
@@ -163,7 +149,7 @@ processors to populate the fields.
Last, but not least, Scrapy comes with some :ref:`commonly used processors
` built-in for convenience.
-.. _this answer on stackoverflow: https://stackoverflow.com/a/35322635
+
Declaring Item Loaders
======================
@@ -271,7 +257,7 @@ There are several ways to modify Item Loader context values:
loader.context['unit'] = 'cm'
2. On Item Loader instantiation (the keyword arguments of Item Loader
- constructor are stored in the Item Loader context)::
+ ``__init__`` method are stored in the Item Loader context)::
loader = ItemLoader(product, unit='cm')
@@ -491,6 +477,8 @@ ItemLoader objects
.. attribute:: item
The :class:`~scrapy.item.Item` object being parsed by this Item Loader.
+ This is mostly used as a property so when attempting to override this
+ value, you may want to check out :attr:`default_item_class` first.
.. attribute:: context
@@ -500,7 +488,7 @@ ItemLoader objects
.. attribute:: default_item_class
An Item class (or factory), used to instantiate items when not given in
- the constructor.
+ the ``__init__`` method.
.. attribute:: default_input_processor
@@ -515,15 +503,15 @@ ItemLoader objects
.. attribute:: default_selector_class
The class used to construct the :attr:`selector` of this
- :class:`ItemLoader`, if only a response is given in the constructor.
- If a selector is given in the constructor this attribute is ignored.
+ :class:`ItemLoader`, if only a response is given in the ``__init__`` method.
+ If a selector is given in the ``__init__`` method this attribute is ignored.
This attribute is sometimes overridden in subclasses.
.. attribute:: selector
The :class:`~scrapy.selector.Selector` object to extract data from.
- It's either the selector given in the constructor or one created from
- the response given in the constructor using the
+ It's either the selector given in the ``__init__`` method or one created from
+ the response given in the ``__init__`` method using the
:attr:`default_selector_class`. This attribute is meant to be
read-only.
@@ -648,7 +636,7 @@ Here is a list of all built-in processors:
.. class:: Identity
The simplest processor, which doesn't do anything. It returns the original
- values unchanged. It doesn't receive any constructor arguments, nor does it
+ values unchanged. It doesn't receive any ``__init__`` method arguments, nor does it
accept Loader contexts.
Example::
@@ -662,7 +650,7 @@ Here is a list of all built-in processors:
Returns the first non-null/non-empty value from the values received,
so it's typically used as an output processor to single-valued fields.
- It doesn't receive any constructor arguments, nor does it accept Loader contexts.
+ It doesn't receive any ``__init__`` method arguments, nor does it accept Loader contexts.
Example::
@@ -673,7 +661,7 @@ Here is a list of all built-in processors:
.. class:: Join(separator=u' ')
- Returns the values joined with the separator given in the constructor, which
+ Returns the values joined with the separator given in the ``__init__`` method, which
defaults to ``u' '``. It doesn't accept Loader contexts.
When using the default separator, this processor is equivalent to the
@@ -711,7 +699,7 @@ Here is a list of all built-in processors:
those which do, this processor will pass the currently active :ref:`Loader
context ` through that parameter.
- The keyword arguments passed in the constructor are used as the default
+ The keyword arguments passed in the ``__init__`` method are used as the default
Loader context values passed to each function call. However, the final
Loader context values passed to functions are overridden with the currently
active Loader context accessible through the :meth:`ItemLoader.context`
@@ -755,12 +743,12 @@ Here is a list of all built-in processors:
['HELLO, 'THIS', 'IS', 'SCRAPY']
As with the Compose processor, functions can receive Loader contexts, and
- constructor keyword arguments are used as default context values. See
+ ``__init__`` method keyword arguments are used as default context values. See
:class:`Compose` processor for more info.
.. class:: SelectJmes(json_path)
- Queries the value using the json path provided to the constructor and returns the output.
+ Queries the value using the json path provided to the ``__init__`` method and returns the output.
Requires jmespath (https://github.com/jmespath/jmespath.py) to run.
This processor takes only one input at a time.
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index 2db0ffddd..dd09477b8 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -255,18 +255,18 @@ scrapy.utils.log module
when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`.
In that case, its usage is not required but it's recommended.
- If you plan on configuring the handlers yourself is still recommended you
- call this function, passing ``install_root_handler=False``. Bear in mind
- there won't be any log output set by default in that case.
+ Another option when running custom scripts is to manually configure the logging.
+ To do this you can use `logging.basicConfig()`_ to set a basic root handler.
- To get you started on manually configuring logging's output, you can use
- `logging.basicConfig()`_ to set a basic root handler. This is an example
- on how to redirect ``INFO`` or higher messages to a file::
+ Note that :class:`~scrapy.crawler.CrawlerProcess` automatically calls ``configure_logging``,
+ so it is recommended to only use `logging.basicConfig()`_ together with
+ :class:`~scrapy.crawler.CrawlerRunner`.
+
+ This is an example on how to redirect ``INFO`` or higher messages to a file::
import logging
from scrapy.utils.log import configure_logging
- configure_logging(install_root_handler=False)
logging.basicConfig(
filename='log.txt',
format='%(levelname)s: %(message)s',
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 431cc6027..206e7cfa5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -441,8 +441,9 @@ See here the methods that you can override in your custom Files Pipeline:
* ``success`` is a boolean which is ``True`` if the image was downloaded
successfully or ``False`` if it failed for some reason
- * ``file_info_or_error`` is a dict containing the following keys (if success
- is ``True``) or a `Twisted Failure`_ if there was a problem.
+ * ``file_info_or_error`` is a dict containing the following keys (if
+ success is ``True``) or a :exc:`~twisted.python.failure.Failure` if
+ there was a problem.
* ``url`` - the url where the file was downloaded from. This is the url of
the request returned from the :meth:`~get_media_requests`
@@ -577,5 +578,4 @@ above::
item['image_paths'] = image_paths
return item
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
.. _MD5 hash: https://en.wikipedia.org/wiki/MD5
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index a6d4f0d6d..e3e8fdc72 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -101,7 +101,7 @@ reactor after ``MySpider`` has finished running.
d.addBoth(lambda _: reactor.stop())
reactor.run() # the script will block here until the crawling is finished
-.. seealso:: `Twisted Reactor Overview`_.
+.. seealso:: :doc:`twisted:core/howto/reactor-basics`
.. _run-multiple-spiders:
@@ -253,6 +253,5 @@ If you are still unable to prevent your bot getting banned, consider contacting
.. _ProxyMesh: https://proxymesh.com/
.. _Google cache: http://www.googleguide.com/cached_pages.html
.. _testspiders: https://github.com/scrapinghub/testspiders
-.. _Twisted Reactor Overview: https://twistedmatrix.com/documents/current/core/howto/reactor-basics.html
.. _Crawlera: https://scrapinghub.com/crawlera
.. _scrapoxy: https://scrapoxy.io/
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 5a76e189e..4cf367d96 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -121,8 +121,8 @@ Request objects
:param errback: a function that will be called if any exception was
raised while processing the request. This includes pages that failed
- with 404 HTTP errors and such. It receives a `Twisted Failure`_ instance
- as first parameter.
+ with 404 HTTP errors and such. It receives a
+ :exc:`~twisted.python.failure.Failure` as first parameter.
For more information,
see :ref:`topics-request-response-ref-errbacks` below.
:type errback: callable
@@ -137,7 +137,7 @@ Request objects
A string containing the URL of this request. Keep in mind that this
attribute contains the escaped URL, so it can differ from the URL passed in
- the constructor.
+ the ``__init__`` method.
This attribute is read-only. To change the URL of a Request use
:meth:`replace`.
@@ -254,8 +254,8 @@ Using errbacks to catch exceptions in request processing
The errback of a request is a function that will be called when an exception
is raise while processing it.
-It receives a `Twisted Failure`_ instance as first parameter and can be
-used to track connection establishment timeouts, DNS errors etc.
+It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
+be used to track connection establishment timeouts, DNS errors etc.
Here's an example spider logging all errors and catching some specific
errors if needed::
@@ -400,7 +400,7 @@ fields with form data from :class:`Response` objects.
.. class:: FormRequest(url, [formdata, ...])
- The :class:`FormRequest` class adds a new keyword parameter to the constructor. The
+ The :class:`FormRequest` class adds a new keyword parameter to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@@ -473,7 +473,7 @@ fields with form data from :class:`Response` objects.
:type dont_click: boolean
The other parameters of this class method are passed directly to the
- :class:`FormRequest` constructor.
+ :class:`FormRequest` ``__init__`` method.
.. versionadded:: 0.10.3
The ``formname`` parameter.
@@ -547,7 +547,7 @@ dealing with JSON requests.
.. class:: JsonRequest(url, [... data, dumps_kwargs])
- The :class:`JsonRequest` class adds two new keyword parameters to the constructor. The
+ The :class:`JsonRequest` class adds two new keyword parameters to the ``__init__`` method. The
remaining arguments are the same as for the :class:`Request` class and are
not documented here.
@@ -556,7 +556,7 @@ dealing with JSON requests.
:param data: is any JSON serializable object that needs to be JSON encoded and assigned to body.
if :attr:`Request.body` argument is provided this parameter will be ignored.
- if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
+ if :attr:`Request.body` argument is not provided and data argument is provided :attr:`Request.method` will be
set to ``'POST'`` automatically.
:type data: JSON serializable object
@@ -721,7 +721,7 @@ TextResponse objects
:class:`Response` class, which is meant to be used only for binary data,
such as images, sounds or any media file.
- :class:`TextResponse` objects support a new constructor argument, in
+ :class:`TextResponse` objects support a new ``__init__`` method argument, in
addition to the base :class:`Response` objects. The remaining functionality
is the same as for the :class:`Response` class and is not documented here.
@@ -755,7 +755,7 @@ TextResponse objects
A string with the encoding of this response. The encoding is resolved by
trying the following mechanisms, in order:
- 1. the encoding passed in the constructor ``encoding`` argument
+ 1. the encoding passed in the ``__init__`` method ``encoding`` argument
2. the encoding declared in the Content-Type HTTP header. If this
encoding is not valid (ie. unknown), it is ignored and the next
@@ -816,5 +816,4 @@ XmlResponse objects
adds encoding auto-discovering support by looking into the XML declaration
line. See :attr:`TextResponse.encoding`.
-.. _Twisted Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
.. _bug in lxml: https://bugs.launchpad.net/lxml/+bug/1665241
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index ff07b9d55..3f29aa323 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -50,10 +50,10 @@ Here is a simple example showing how you can catch signals and perform some acti
Deferred signal handlers
========================
-Some signals support returning `Twisted deferreds`_ from their handlers, see
-the :ref:`topics-signals-ref` below to know which ones.
+Some signals support returning :class:`~twisted.internet.defer.Deferred`
+objects from their handlers, see the :ref:`topics-signals-ref` below to know
+which ones.
-.. _Twisted deferreds: https://twistedmatrix.com/documents/current/core/howto/defer.html
.. _topics-signals-ref:
@@ -155,8 +155,8 @@ item_error
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.spiders.Spider` object
- :param failure: the exception raised as a Twisted `Failure`_ object
- :type failure: `Failure`_ object
+ :param failure: the exception raised
+ :type failure: twisted.python.failure.Failure
spider_closed
-------------
@@ -236,8 +236,8 @@ spider_error
This signal does not support returning deferreds from their handlers.
- :param failure: the exception raised as a Twisted `Failure`_ object
- :type failure: `Failure`_ object
+ :param failure: the exception raised
+ :type failure: twisted.python.failure.Failure
:param response: the response being processed when the exception was raised
:type response: :class:`~scrapy.http.Response` object
@@ -333,5 +333,3 @@ response_downloaded
:param spider: the spider for which the response is intended
:type spider: :class:`~scrapy.spiders.Spider` object
-
-.. _Failure: https://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 3bef20bf3..da7a0022b 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python
-from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
diff --git a/pytest.ini b/pytest.ini
index 8c5a2cd54..f088e10ef 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -7,10 +7,7 @@ addopts =
--doctest-modules
--ignore=docs/_ext
--ignore=docs/conf.py
- --ignore=docs/intro/tutorial.rst
--ignore=docs/news.rst
- --ignore=docs/topics/commands.rst
- --ignore=docs/topics/debug.rst
--ignore=docs/topics/developer-tools.rst
--ignore=docs/topics/dynamic-content.rst
--ignore=docs/topics/items.rst
@@ -23,250 +20,230 @@ addopts =
--ignore=docs/utils
twisted = 1
flake8-ignore =
+ # Files that are only meant to provide top-level imports are expected not
+ # to use any of their imports:
+ scrapy/core/downloader/handlers/http.py F401
+ scrapy/http/__init__.py F401
+ # Issues pending a review:
# extras
- extras/qps-bench-server.py E261 E501
- extras/qpsclient.py E501 E261 E501
+ extras/qps-bench-server.py E501
+ extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
- scrapy/commands/check.py F401 E501
+ scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
- scrapy/commands/fetch.py E401 E302 E501 E128 E502 E731
+ scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
- scrapy/commands/list.py E302
scrapy/commands/parse.py E128 E501 E731 E226
scrapy/commands/runspider.py E501
- scrapy/commands/settings.py E302 E128
+ scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
- scrapy/commands/startproject.py E502 E127 E501 E128
+ scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
- scrapy/commands/view.py F401 E302
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
- scrapy/contracts/default.py E502 E128
+ scrapy/contracts/default.py E128
# scrapy/core
- scrapy/core/engine.py E261 E501 E128 E127 E306 E502
+ scrapy/core/engine.py E501 E128 E127 E306 E502
scrapy/core/scheduler.py E501
- scrapy/core/scraper.py E501 E306 E261 E128 W504
- scrapy/core/spidermw.py E501 E731 E502 E231 E126 E226
- scrapy/core/downloader/__init__.py F401 E501
+ scrapy/core/scraper.py E501 E306 E128 W504
+ scrapy/core/spidermw.py E501 E731 E126 E226
+ scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E305 E241
- scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
+ scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
- scrapy/core/downloader/handlers/http.py F401
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
- scrapy/core/downloader/handlers/s3.py E501 F401 E502 E128 E126
+ scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
- scrapy/downloadermiddlewares/ajaxcrawl.py E302 E501 E226
+ scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
- scrapy/downloadermiddlewares/httpcompression.py E502 E128
+ scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
- scrapy/downloadermiddlewares/robotstxt.py F401 E501
+ scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
- scrapy/extensions/closespider.py E501 E502 E128 E123
- scrapy/extensions/corestats.py E302 E501
+ scrapy/extensions/closespider.py E501 E128 E123
+ scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
- scrapy/extensions/httpcache.py E128 E501 E303 F401
+ scrapy/extensions/httpcache.py E128 E501 E303
scrapy/extensions/memdebug.py E501
- scrapy/extensions/spiderstate.py E302 E501
+ scrapy/extensions/spiderstate.py E501
scrapy/extensions/telnet.py E501 W504
scrapy/extensions/throttle.py E501
# scrapy/http
- scrapy/http/__init__.py F401
scrapy/http/common.py E501
scrapy/http/cookies.py E501
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
scrapy/http/response/__init__.py E501 E128 W293 W291
- scrapy/http/response/html.py E302
scrapy/http/response/text.py E501 W293 E128 E124
- scrapy/http/response/xml.py E302
# scrapy/linkextractors
- scrapy/linkextractors/__init__.py E731 E502 E501 E402 F401
+ scrapy/linkextractors/__init__.py E731 E501 E402
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
# scrapy/loader
- scrapy/loader/__init__.py E501 E502 E128
- scrapy/loader/common.py E302
+ scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
- scrapy/pipelines/__init__.py E302
scrapy/pipelines/files.py E116 E501 E266
scrapy/pipelines/images.py E265 E501
scrapy/pipelines/media.py E125 E501 E266
# scrapy/selector
- scrapy/selector/__init__.py F403 F401
- scrapy/selector/unified.py F401 E501 E111
+ scrapy/selector/__init__.py F403
+ scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
- scrapy/settings/default_settings.py E501 E261 E114 E116 E226
+ scrapy/settings/default_settings.py E501 E114 E116 E226
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
scrapy/spidermiddlewares/offsite.py E501
- scrapy/spidermiddlewares/referer.py F401 E501 E129 W503 W504
+ scrapy/spidermiddlewares/referer.py E501 E129 W503 W504
scrapy/spidermiddlewares/urllength.py E501
# scrapy/spiders
- scrapy/spiders/__init__.py F401 E501 E402
+ scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
- scrapy/spiders/feed.py E501 E261
+ scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/benchserver.py E501
- scrapy/utils/boto.py F401
- scrapy/utils/conf.py E402 E502 E501
- scrapy/utils/console.py E302 E261 F401 E306 E305
- scrapy/utils/curl.py F401
+ scrapy/utils/conf.py E402 E501
+ scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
- scrapy/utils/decorators.py E501 E302
- scrapy/utils/defer.py E501 E302 E128
+ scrapy/utils/decorators.py E501
+ scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
- scrapy/utils/display.py E302
- scrapy/utils/engine.py F401 E261 E302
- scrapy/utils/ftp.py E302
- scrapy/utils/gz.py E305 E501 E302 W504
- scrapy/utils/http.py F403 F401 E226
- scrapy/utils/httpobj.py E302 E501
+ scrapy/utils/gz.py E305 E501 W504
+ scrapy/utils/http.py F403 E226
+ scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
- scrapy/utils/job.py E302
scrapy/utils/log.py E128 W503
- scrapy/utils/markup.py F403 F401 W292
+ scrapy/utils/markup.py F403 W292
scrapy/utils/misc.py E501 E226
- scrapy/utils/multipart.py F403 F401 W292
+ scrapy/utils/multipart.py F403 W292
scrapy/utils/project.py E501
- scrapy/utils/python.py E501 E302
- scrapy/utils/reactor.py E302 E226
+ scrapy/utils/python.py E501
+ scrapy/utils/reactor.py E226
scrapy/utils/reqser.py E501
- scrapy/utils/request.py E302 E127 E501
- scrapy/utils/response.py E501 E302 E128
+ scrapy/utils/request.py E127 E501
+ scrapy/utils/response.py E501 E128
scrapy/utils/signal.py E501 E128
scrapy/utils/sitemap.py E501
- scrapy/utils/spider.py E271 E302 E501
+ scrapy/utils/spider.py E271 E501
scrapy/utils/ssl.py E501
- scrapy/utils/template.py E302
- scrapy/utils/test.py E302 E501
- scrapy/utils/url.py E501 F403 F401 E128 F405
+ scrapy/utils/test.py E501
+ scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
scrapy/_monkeypatches.py W293
- scrapy/cmdline.py E502 E501
+ scrapy/cmdline.py E501
scrapy/crawler.py E501
- scrapy/dupefilters.py E302 E501 E202
- scrapy/exceptions.py E302 E501
- scrapy/exporters.py E501 E261 E226
- scrapy/extension.py E302
- scrapy/interfaces.py E302 E501
+ scrapy/dupefilters.py E501 E202
+ scrapy/exceptions.py E501
+ scrapy/exporters.py E501 E226
+ scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
scrapy/logformatter.py E501 W293
scrapy/mail.py E402 E128 E501 E502
- scrapy/middleware.py E502 E128 E501
+ scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
- scrapy/resolver.py E302
scrapy/responsetypes.py E128 E501 E305
- scrapy/robotstxt.py E302 E501
+ scrapy/robotstxt.py E501
scrapy/shell.py E501
scrapy/signalmanager.py E501
scrapy/spiderloader.py E225 F841 E501 E126
scrapy/squeues.py E128
scrapy/statscollectors.py E501
# tests
- tests/__init__.py F401 E402 E501
- tests/mockserver.py E401 E501 E126 E123 F401
- tests/pipelines.py E302 F841 E226
- tests/spiders.py E302 E501 E127
+ tests/__init__.py E402 E501
+ tests/mockserver.py E401 E501 E126 E123
+ tests/pipelines.py F841 E226
+ tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
- tests/test_command_fetch.py E501 E261
- tests/test_command_parse.py F401 E302 E501 E128 E303 E226
+ tests/test_command_fetch.py E501
+ tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
- tests/test_commands.py F401 E128 E501
+ tests/test_commands.py E128 E501
tests/test_contracts.py E501 E128 W293
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
- tests/test_dependencies.py E302 F841 E501 E305
- tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 F401 E501 E502 E701 E711 E126 E226 E123
+ tests/test_dependencies.py F841 E501 E305
+ tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
- tests/test_downloadermiddleware_ajaxcrawlable.py E302 E501
+ tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
tests/test_downloadermiddleware_decompression.py E127
tests/test_downloadermiddleware_defaultheaders.py E501
tests/test_downloadermiddleware_downloadtimeout.py E501
- tests/test_downloadermiddleware_httpcache.py E713 E501 E302 E305 F401
- tests/test_downloadermiddleware_httpcompression.py E501 F401 E251 E126 E123
- tests/test_downloadermiddleware_httpproxy.py F401 E501 E128
+ tests/test_downloadermiddleware_httpcache.py E501 E305
+ tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
+ tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
- tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
+ tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
- tests/test_dupefilters.py E302 E221 E501 E741 W293 W291 E128 E124
- tests/test_engine.py E401 E501 E502 E128 E261
+ tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
+ tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E306 E128 E124
- tests/test_extension_telnet.py F401 F841
- tests/test_feedexport.py E501 F401 F841 E241
+ tests/test_extension_telnet.py F841
+ tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
- tests/test_http_headers.py E302 E501
- tests/test_http_request.py F401 E402 E501 E231 E261 E127 E128 W293 E502 E128 E502 E126 E123
- tests/test_http_response.py E501 E301 E502 E128 E265
- tests/test_item.py E701 E128 E231 F841 E306
+ tests/test_http_headers.py E501
+ tests/test_http_request.py E402 E501 E127 E128 W293 E128 E126 E123
+ tests/test_http_response.py E501 E301 E128 E265
+ tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
- tests/test_linkextractors.py E501 E128 E231 E124
- tests/test_loader.py E302 E501 E731 E303 E741 E128 E117 E241
- tests/test_logformatter.py E128 E501 E231 E122 E302
- tests/test_mail.py E302 E128 E501 E305
- tests/test_middleware.py E302 E501 E128
+ tests/test_linkextractors.py E501 E128 E124
+ tests/test_loader.py E501 E731 E303 E741 E128 E117 E241
+ tests/test_logformatter.py E128 E501 E122
+ tests/test_mail.py E128 E501 E305
+ tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
- tests/test_pipeline_files.py F401 E501 W293 E303 E272 E226
- tests/test_pipeline_images.py F401 F841 E501 E303
- tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
+ tests/test_pipeline_files.py E501 W293 E303 E272 E226
+ tests/test_pipeline_images.py F841 E501 E303
+ tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
+ tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
- tests/test_responsetypes.py E501 E302 E305
- tests/test_robotstxt_interface.py F401 E302 E501 W291 E501
+ tests/test_responsetypes.py E501 E305
+ tests/test_robotstxt_interface.py E501 W291 E501
tests/test_scheduler.py E501 E126 E123
- tests/test_selector.py F401 E501 E127
- tests/test_spider.py E501 F401
+ tests/test_selector.py E501 E127
+ tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
- tests/test_spidermiddleware_offsite.py E302 E501 E128 E111 W293
- tests/test_spidermiddleware_output_chain.py F401 E501 E302 W293 E226
- tests/test_spidermiddleware_referer.py F401 E501 E302 F841 E125 E201 E261 E124 E501 E241 E121
- tests/test_squeues.py E501 E302 E701 E741
- tests/test_utils_conf.py E501 E231 E303 E128
- tests/test_utils_console.py E302 E231
+ tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
+ tests/test_spidermiddleware_output_chain.py E501 W293 E226
+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
+ tests/test_squeues.py E501 E701 E741
+ tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
- tests/test_utils_defer.py E306 E261 E501 E302 F841 E226
+ tests/test_utils_defer.py E306 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
- tests/test_utils_http.py E302 E501 E502 E128 W504
- tests/test_utils_httpobj.py E302
- tests/test_utils_iterators.py E501 E128 E129 E302 E303 E241
+ tests/test_utils_http.py E501 E128 W504
+ tests/test_utils_iterators.py E501 E128 E129 E303 E241
tests/test_utils_log.py E741 E226
tests/test_utils_python.py E501 E303 E731 E701 E305
- tests/test_utils_reqser.py F401 E501 E128
- tests/test_utils_request.py E302 E501 E128 E305
+ tests/test_utils_reqser.py E501 E128
+ tests/test_utils_request.py E501 E128 E305
tests/test_utils_response.py E501
- tests/test_utils_signal.py E741 F841 E302 E731 E226
- tests/test_utils_sitemap.py E302 E128 E501 E124
- tests/test_utils_spider.py E261 E302 E305
+ tests/test_utils_signal.py E741 F841 E731 E226
+ tests/test_utils_sitemap.py E128 E501 E124
+ tests/test_utils_spider.py E305
tests/test_utils_template.py E305
- tests/test_utils_url.py F401 E501 E127 E302 E305 E211 E125 E501 E226 E241 E126 E123
+ tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
- tests/mocks/dummydbm.py E302
- tests/test_cmdline/__init__.py E502 E501
- tests/test_cmdline/extensions.py E302
- tests/test_settings/__init__.py F401 E501 E128
- tests/test_spiderloader/__init__.py E128 E501 E302
- tests/test_spiderloader/test_spiders/spider0.py E302
- tests/test_spiderloader/test_spiders/spider1.py E302
- tests/test_spiderloader/test_spiders/spider2.py E302
- tests/test_spiderloader/test_spiders/spider3.py E302
- tests/test_spiderloader/test_spiders/nested/spider4.py E302
- tests/test_utils_misc/__init__.py E501 E231
+ tests/test_cmdline/__init__.py E501
+ tests/test_settings/__init__.py E501 E128
+ tests/test_spiderloader/__init__.py E128 E501
+ tests/test_utils_misc/__init__.py E501
diff --git a/requirements-py3.txt b/requirements-py3.txt
deleted file mode 100644
index 2c98e6f6d..000000000
--- a/requirements-py3.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-parsel>=1.5.0
-PyDispatcher>=2.0.5
-Twisted>=17.9.0
-w3lib>=1.17.0
-protego>=0.1.15
-
-pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0 # Earlier versions would fail to install
-
-# Reference versions taken from
-# https://packages.ubuntu.com/xenial/python/
-# https://packages.ubuntu.com/xenial/zope/
-cssselect>=0.9.1
-lxml>=3.5.0
-service_identity>=16.0.0
-six>=1.10.0
-zope.interface>=4.1.3
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 230e5cee3..fb8357f3c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index b68099cad..f74f89bda 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,14 +1,4 @@
-import six
-from six.moves import copyreg
-
-
-if six.PY2:
- from urlparse import urlparse
-
- # workaround for https://bugs.python.org/issue9374 - Python < 2.7.4
- if urlparse('s3://bucket/key?key=value').query != 'key=value':
- from urlparse import uses_query
- uses_query.append('s3')
+import copyreg
# Undo what Twisted's perspective broker adds to pickle register
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..ec78f7c91 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import sys
import os
import optparse
@@ -68,7 +67,7 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
if inproject:
- print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
+ print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
@@ -124,7 +123,7 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
- parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
+ parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 90c8d56a2..7bbe362e7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,8 +1,7 @@
import sys
import time
import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
import scrapy
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index 3e6c11b7d..9d4437a47 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,6 +1,4 @@
-from __future__ import print_function
import time
-import sys
from collections import defaultdict
from unittest import TextTestRunner, TextTestResult as _TextTestResult
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 7d4840529..0e149941d 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,5 +1,4 @@
-from __future__ import print_function
-import sys, six
+import sys
from w3lib.url import is_url
from scrapy.commands import ScrapyCommand
@@ -8,6 +7,7 @@ from scrapy.exceptions import UsageError
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
+
class Command(ScrapyCommand):
requires_project = False
@@ -24,12 +24,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("--spider", dest="spider",
- help="use this spider")
- parser.add_option("--headers", dest="headers", action="store_true", \
- help="print response HTTP headers instead of body")
- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
- default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+ parser.add_option("--spider", dest="spider", help="use this spider")
+ parser.add_option("--headers", dest="headers", action="store_true",
+ help="print response HTTP headers instead of body")
+ parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
+ default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
@@ -45,8 +44,7 @@ class Command(ScrapyCommand):
self._print_bytes(response.body)
def _print_bytes(self, bytes_):
- bytes_writer = sys.stdout if six.PY2 else sys.stdout.buffer
- bytes_writer.write(bytes_ + b'\n')
+ sys.stdout.buffer.write(bytes_ + b'\n')
def run(self, args, opts):
if len(args) != 1 or not is_url(args[0]):
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d5498bb5c..adb01fa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import os
import shutil
import string
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index a255b3b94..54d7bb228 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,6 +1,6 @@
-from __future__ import print_function
from scrapy.commands import ScrapyCommand
+
class Command(ScrapyCommand):
requires_project = True
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ef8acd29c..ff6f1d8cd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import json
import logging
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index bee52f06a..603bafb9f 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,9 +1,9 @@
-from __future__ import print_function
import json
from scrapy.commands import ScrapyCommand
from scrapy.settings import BaseSettings
+
class Command(ScrapyCommand):
requires_project = False
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 3b9f6eabb..b123e5c84 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import re
import os
import string
@@ -44,8 +43,8 @@ class Command(ScrapyCommand):
return False
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
- print('Error: Project names must begin with a letter and contain'\
- ' only\nletters, numbers and underscores')
+ print('Error: Project names must begin with a letter and contain'
+ ' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
else:
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 8651948f7..1516c5997 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index 59e665016..41e77ba3b 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,6 +1,7 @@
-from scrapy.commands import fetch, ScrapyCommand
+from scrapy.commands import fetch
from scrapy.utils.response import open_in_browser
+
class Command(fetch.Command):
def short_desc(self):
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 24f6c2e77..3002fc702 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -4,7 +4,7 @@ from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
-from . import Contract
+from scrapy.contracts import Contract
# contracts
@@ -86,8 +86,8 @@ class ReturnsContract(Contract):
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
- raise ContractFail("Returned %s %s, expected %s" % \
- (occurrences, self.obj_name, expected))
+ raise ContractFail("Returned %s %s, expected %s" %
+ (occurrences, self.obj_name, expected))
class ScrapesContract(Contract):
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 949dacbc8..157dc3418 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,19 +1,16 @@
-from __future__ import absolute_import
import random
-import warnings
from time import time
from datetime import datetime
from collections import deque
-import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
@@ -190,7 +187,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
- for slot in six.itervalues(self.slots):
+ for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 89d2776ae..6e023ebcc 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -67,15 +67,18 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
"""
Twisted-recommended context factory for web clients.
- Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
- "The default is to use a BrowserLikePolicyForHTTPS,
- so unless you have special requirements you can leave this as-is."
+ Quoting the documentation of the :class:`~twisted.web.client.Agent` class:
- creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
- except this context factory allows setting the TLS/SSL method to use.
+ The default is to use a
+ :class:`~twisted.web.client.BrowserLikePolicyForHTTPS`, so unless you
+ have special requirements you can leave this as-is.
- Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
- which allows TLS protocol negotiation.
+ :meth:`creatorForNetloc` is the same as
+ :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
+ factory allows setting the TLS/SSL method to use.
+
+ The default OpenSSL method is ``TLS_METHOD`` (also called
+ ``SSLv23_METHOD``) which allows TLS protocol negotiation.
"""
def creatorForNetloc(self, hostname, port):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..39a0b1f51 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,8 +1,9 @@
"""Download handlers for different schemes"""
import logging
+
from twisted.internet import defer
-import six
+
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
- for scheme, clspath in six.iteritems(handlers):
+ for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index ad25beb3b..9e5020753 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -17,8 +17,8 @@ class DataURIDownloadHandler(object):
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
- if (issubclass(respcls, TextResponse) and
- uri.media_type.split('/')[0] == 'text'):
+ if (issubclass(respcls, TextResponse)
+ and uri.media_type.split('/')[0] == 'text'):
charset = uri.media_type_parameters.get('charset')
resp_kwargs['encoding'] = charset
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 39ed67a1a..aef231e82 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index ac4b867c3..52535bd8b 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,3 +1,4 @@
-from __future__ import absolute_import
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+ HTTP11DownloadHandler as HTTPDownloadHandler,
+)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 91b45a8fc..1212feb79 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -2,10 +2,10 @@
import re
import logging
+import warnings
from io import BytesIO
from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
from zope.interface import implementer
from twisted.internet import defer, reactor, protocol
@@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
from twisted.internet.endpoints import TCP4ClientEndpoint
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
@@ -174,7 +175,7 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
r"""
Return binary content of a CONNECT request.
- >>> from scrapy.utils.python import to_native_str as s
+ >>> from scrapy.utils.python import to_unicode as s
>>> s(tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
@@ -285,6 +286,12 @@ class ScrapyAgent(object):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
+ if omitConnectTunnel:
+ warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
+ "If you use Crawlera, it doesn't require this mode anymore, "
+ "so you should update scrapy-crawlera to 1.3.0+ "
+ "and remove '?noconnect' from the Crawlera URL.",
+ ScrapyDeprecationWarning)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index d8bbdd326..d6fbd54ee 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,9 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
def _get_boto_connection():
@@ -21,7 +21,7 @@ def _get_boto_connection():
return http_request.headers
try:
- import boto.auth
+ import boto.auth # noqa: F401
except ImportError:
_S3Connection = _v19_S3Connection
else:
@@ -32,8 +32,8 @@ def _get_boto_connection():
class S3DownloadHandler(object):
- def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
- httpdownloadhandler=HTTPDownloadHandler, **kw):
+ def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
+ httpdownloadhandler=HTTPDownloadHandler, **kw):
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 7a6a4dfac..38608a429 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,6 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
-import six
-
from twisted.internet import defer
from scrapy.exceptions import _InvalidOutput
@@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
+ (method.__self__.__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, response=response, spider=spider)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response)))
+ (method.__self__.__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, exception=exception, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response)))
+ (method.__self__.__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3fe13414a..fc796e8bb 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,5 @@
from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
@@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
def connectionMade(self):
- self.headers = Headers() # bucket for response headers
+ self.headers = Headers() # bucket for response headers
# Method command
self.sendCommand(self.factory.method, self.factory.path)
@@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
if self.factory.url.startswith(b'https'):
self.transport.stopProducing()
- self.factory.noPage(\
- defer.TimeoutError("Getting %s took longer than %s seconds." % \
- (self.factory.url, self.factory.timeout)))
+ self.factory.noPage(
+ defer.TimeoutError("Getting %s took longer than %s seconds." %
+ (self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index fa913e528..829e69993 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -25,7 +25,7 @@ class Slot(object):
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
- self.inprogress = set() # requests in progress
+ self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 40de6b87a..b3d585cce 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -123,7 +123,7 @@ class Scraper(object):
callback/errback"""
assert isinstance(response, (Response, Failure))
- dfd = self._scrape2(response, request, spider) # returns spiders processed output
+ dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
return dfd
@@ -231,9 +231,9 @@ class Scraper(object):
signal=signals.item_dropped, item=item, response=response,
spider=spider, exception=output.value)
else:
- logger.error('Error processing %(item)s', {'item': item},
- exc_info=failure_to_exc_info(output),
- extra={'spider': spider})
+ logkws = self.logformatter.error(item, ex, response, spider)
+ logger.log(*logformatter_adapter(logkws), extra={'spider': spider},
+ exc_info=failure_to_exc_info(output))
return self.signals.send_catch_log_deferred(
signal=signals.item_error, item=item, response=response,
spider=spider, failure=output)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b5f9837ff..097a374bf 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
"""
from itertools import chain, islice
-import six
from twisted.python.failure import Failure
from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
@@ -36,16 +35,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
self.methods['process_spider_exception'].appendleft(getattr(mw, 'process_spider_exception', None))
def scrape_response(self, scrape_func, response, request, spider):
- fname = lambda f:'%s.%s' % (
- six.get_method_self(f).__class__.__name__,
- six.get_method_function(f).__name__)
+ fname = lambda f: '%s.%s' % (
+ f.__self__.__class__.__name__,
+ f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
if result is not None:
- raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
+ raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
.format(fname(method), type(result)))
except _InvalidOutput:
raise
@@ -70,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
- raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
+ raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
.format(fname(method), type(result)))
return _failure
@@ -104,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = evaluate_iterable(result, method_index)
else:
- raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
+ raise _InvalidOutput('Middleware {} must return an iterable, got {}'
.format(fname(method), type(result)))
return chain(result, recovered)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ded3c082b..6c7eb737b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,9 +1,8 @@
-import six
-import signal
import logging
+import pprint
+import signal
import warnings
-import sys
from twisted.internet import reactor, defer
from zope.interface.verify import verifyClass, DoesNotImplement
@@ -22,6 +21,7 @@ from scrapy.utils.log import (
get_scrapy_root_handler, install_scrapy_root_handler)
from scrapy import signals
+
logger = logging.getLogger(__name__)
@@ -46,7 +46,8 @@ class Crawler(object):
logging.root.addHandler(handler)
d = dict(overridden_settings(self.settings))
- logger.info("Overridden settings: %(settings)r", {'settings': d})
+ logger.info("Overridden settings:\n%(settings)s",
+ {'settings': pprint.pformat(d)})
if get_scrapy_root_handler() is not None:
# scrapy root handler already installed: update it with new settings
@@ -88,20 +89,9 @@ class Crawler(object):
yield self.engine.open_spider(self.spider, start_requests)
yield defer.maybeDeferred(self.engine.start)
except Exception:
- # In Python 2 reraising an exception after yield discards
- # the original traceback (see https://bugs.python.org/issue7563),
- # so sys.exc_info() workaround is used.
- # This workaround also works in Python 3, but it is not needed,
- # and it is slower, so in Python 3 we use native `raise`.
- if six.PY2:
- exc_info = sys.exc_info()
-
self.crawling = False
if self.engine is not None:
yield self.engine.close()
-
- if six.PY2:
- six.reraise(*exc_info)
raise
def _create_spider(self, *args, **kwargs):
@@ -122,7 +112,7 @@ class Crawler(object):
class CrawlerRunner(object):
"""
This is a convenient helper class that keeps track of, manages and runs
- crawlers inside an already setup Twisted `reactor`_.
+ crawlers inside an already setup :mod:`~twisted.internet.reactor`.
The CrawlerRunner object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
@@ -216,7 +206,7 @@ class CrawlerRunner(object):
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
- if isinstance(spidercls, six.string_types):
+ if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
@@ -245,12 +235,13 @@ class CrawlerProcess(CrawlerRunner):
A class to run multiple scrapy crawlers in a process simultaneously.
This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support
- for starting a Twisted `reactor`_ and handling shutdown signals, like the
- keyboard interrupt command Ctrl-C. It also configures top-level logging.
+ for starting a :mod:`~twisted.internet.reactor` and handling shutdown
+ signals, like the keyboard interrupt command Ctrl-C. It also configures
+ top-level logging.
This utility should be a better fit than
:class:`~scrapy.crawler.CrawlerRunner` if you aren't running another
- Twisted `reactor`_ within your application.
+ :mod:`~twisted.internet.reactor` within your application.
The CrawlerProcess object must be instantiated with a
:class:`~scrapy.settings.Settings` object.
@@ -285,9 +276,9 @@ class CrawlerProcess(CrawlerRunner):
def start(self, stop_after_crawl=True):
"""
- This method starts a Twisted `reactor`_, adjusts its pool size to
- :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache based
- on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
+ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
+ size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
+ based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`.
If ``stop_after_crawl`` is True, the reactor will be stopped after all
crawlers have finished, using :meth:`join`.
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 72715dba7..7a140fcad 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import re
import logging
-import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@@ -67,7 +65,9 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r''))
+_ajax_crawlable_re = re.compile(r'')
+
+
def _has_ajaxcrawlable_meta(text):
"""
>>> _has_ajaxcrawlable_meta('value ')
+ r = self.response_class("http://example.com", encoding='utf-8',
+ body=b'\xf0value ')
assert u'value ' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
@@ -534,7 +532,7 @@ class XmlResponseTest(TextResponseTest):
r2 = self.response_class("http://www.example.com", body=body)
self._assert_response_values(r2, 'iso-8859-1', body)
- # make sure replace() preserves the explicit encoding passed in the constructor
+ # make sure replace() preserves the explicit encoding passed in the __init__ method
body = b"""Page title<title>
- <body><p><a href="item/12.html">Item 12</a></p>
- <p><a href="/about.html">About us</a></p>
- <img src="/logo.png" alt="Company logo (not a link)" />
- <p><a href="../othercat.html">Other category</a></p>
- <p><a href="/">>></a></p>
- <p><a href="/" /></p>
- </body></html>"""
- response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
- lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
- Link(url='http://example.org/about.html', text='About us'),
- Link(url='http://example.org/othercat.html', text='Other category'),
- Link(url='http://example.org/', text='>>'),
- Link(url='http://example.org/', text='')])
-
- def test_base_url(self):
- html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
- <body><p><a href="item/12.html">Item 12</a></p>
- </body></html>"""
- response = HtmlResponse("http://example.org/somepage/index.html", body=html)
-
- lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
- self.assertEqual(lx.extract_links(response),
- [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
-
- # base url is an absolute path and relative to host
- html = """<html><head><title>Page title<title><base href="/" />
- <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
- response = HtmlResponse("https://example.org/somepage/index.html", body=html)
- self.assertEqual(lx.extract_links(response),
- [Link(url='https://example.org/item/12.html', text='Item 12')])
-
- # base url has no scheme
- html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
- <body><p><a href="item/12.html">Item 12</a></p></body></html>"""
- response = HtmlResponse("https://example.org/somepage/index.html", body=html)
- self.assertEqual(lx.extract_links(response),
- [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
-
- def test_link_text_wrong_encoding(self):
- html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
- response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
- lx = BaseSgmlLinkExtractor()
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
- ])
-
- def test_extraction_encoding(self):
- body = get_testdata('link_extractor', 'linkextractor_noenc.html')
- response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
- response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
- body = get_testdata('link_extractor', 'linkextractor_latin1.html')
- response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
-
- lx = BaseSgmlLinkExtractor()
- self.assertEqual(lx.extract_links(response_utf8), [
- Link(url='http://example.com/sample_%C3%B1.html', text=''),
- Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
- ])
-
- self.assertEqual(lx.extract_links(response_noenc), [
- Link(url='http://example.com/sample_%C3%B1.html', text=''),
- Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
- ])
-
- # document encoding does not affect URL path component, only query part
- # >>> u'sample_ñ.html'.encode('utf8')
- # b'sample_\xc3\xb1.html'
- # >>> u"sample_á.html".encode('utf8')
- # b'sample_\xc3\xa1.html'
- # >>> u"sample_ö.html".encode('utf8')
- # b'sample_\xc3\xb6.html'
- # >>> u"£32".encode('latin1')
- # b'\xa332'
- # >>> u"µ".encode('latin1')
- # b'\xb5'
- self.assertEqual(lx.extract_links(response_latin1), [
- Link(url='http://example.com/sample_%C3%B1.html', text=''),
- Link(url='http://example.com/sample_%C3%A1.html', text='sample \xe1 text'.decode('latin1')),
- Link(url='http://example.com/sample_%C3%B6.html?price=%A332&%B5=unit', text=''),
- ])
-
- def test_matches(self):
- url1 = 'http://lotsofstuff.com/stuff1/index'
- url2 = 'http://evenmorestuff.com/uglystuff/index'
-
- lx = BaseSgmlLinkExtractor()
- self.assertEqual(lx.matches(url1), True)
- self.assertEqual(lx.matches(url2), True)
-
-
-class HtmlParserLinkExtractorTestCase(unittest.TestCase):
-
- def setUp(self):
- body = get_testdata('link_extractor', 'sgml_linkextractor.html')
- self.response = HtmlResponse(url='http://example.com/index', body=body)
-
- def test_extraction(self):
- # Default arguments
- lx = HtmlParserLinkExtractor()
- self.assertEqual(lx.extract_links(self.response), [
- Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
- Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=u''),
- Link(url='http://example.com/innertag.html', text=u'inner tag'),
- Link(url='http://example.com/page%204.html', text=u'href with whitespaces'),
- ])
-
- def test_link_wrong_href(self):
- html = """
- <a href="http://example.org/item1.html">Item 1</a>
- <a href="http://[example.org/item2.html">Item 2</a>
- <a href="http://example.org/item3.html">Item 3</a>
- """
- response = HtmlResponse("http://example.org/index.html", body=html)
- lx = HtmlParserLinkExtractor()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
- ])
-
-
-class SgmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
- extractor_cls = SgmlLinkExtractor
- escapes_whitespace = True
-
- def test_deny_extensions(self):
- html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
- response = HtmlResponse("http://example.org/", body=html)
- lx = SgmlLinkExtractor(deny_extensions="jpg")
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.org/page.html', text=u'asd'),
- ])
-
- def test_attrs_sgml(self):
- html = """<html><area href="sample1.html"></area>
- <a ref="sample2.html">sample text 2</a></html>"""
- response = HtmlResponse("http://example.com/index.html", body=html)
- lx = SgmlLinkExtractor(attrs="href")
- self.assertEqual(lx.extract_links(response), [
- Link(url='http://example.com/sample1.html', text=u''),
- ])
-
- def test_link_nofollow(self):
- html = """
- <a href="page.html?action=print" rel="nofollow">Printer-friendly page</a>
- <a href="about.html">About us</a>
- <a href="http://google.com/something" rel="external nofollow">Something</a>
- """
- response = HtmlResponse("http://example.org/page.html", body=html)
- lx = SgmlLinkExtractor()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
- Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
- Link(url='http://google.com/something', text=u'Something', nofollow=True),
- ])
-
-
-class RegexLinkExtractorTestCase(unittest.TestCase):
- # XXX: RegexLinkExtractor is not deprecated yet, but it must be rewritten
- # not to depend on SgmlLinkExractor. Its speed is also much worse
- # than it should be.
-
- def setUp(self):
- body = get_testdata('link_extractor', 'sgml_linkextractor.html')
- self.response = HtmlResponse(url='http://example.com/index', body=body)
-
- def test_extraction(self):
- # Default arguments
- lx = RegexLinkExtractor()
- self.assertEqual(lx.extract_links(self.response),
- [Link(url='http://example.com/sample2.html', text=u'sample 2'),
- Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
- Link(url='http://example.com/sample3.html#foo', text=u'sample 3 repetition with fragment'),
- Link(url='http://www.google.com/something', text=u''),
- Link(url='http://example.com/innertag.html', text=u'inner tag'),])
-
- def test_link_wrong_href(self):
- html = """
- <a href="http://example.org/item1.html">Item 1</a>
- <a href="http://[example.org/item2.html">Item 2</a>
- <a href="http://example.org/item3.html">Item 3</a>
- """
- response = HtmlResponse("http://example.org/index.html", body=html)
- lx = RegexLinkExtractor()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://example.org/item1.html', text=u'Item 1', nofollow=False),
- Link(url='http://example.org/item3.html', text=u'Item 3', nofollow=False),
- ])
-
- def test_html_base_href(self):
- html = """
- <html>
- <head>
- <base href="http://b.com/">
- </head>
- <body>
- <a href="test.html"></a>
- </body>
- </html>
- """
- response = HtmlResponse("http://a.com/", body=html)
- lx = RegexLinkExtractor()
- self.assertEqual([link for link in lx.extract_links(response)], [
- Link(url='http://b.com/test.html', text=u'', nofollow=False),
- ])
-
- @unittest.expectedFailure
- def test_extraction(self):
- # RegexLinkExtractor doesn't parse URLs with leading/trailing
- # whitespaces correctly.
- super(RegexLinkExtractorTestCase, self).test_extraction()
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 4a4264a2a..579a85ff6 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,8 +1,6 @@
from functools import partial
import unittest
-import six
-
from scrapy.http import HtmlResponse
from scrapy.item import Item, Field
from scrapy.loader import ItemLoader
@@ -157,7 +155,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_get_value(self):
il = NameItemLoader()
- self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
+ self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
@@ -258,7 +256,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
- name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
+ name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
il.add_value('name', u'marta')
@@ -266,7 +264,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_extend_default_input_processors(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
- name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
+ name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
il = ChildDefaultedItemLoader()
il.add_value('name', u'marta')
@@ -689,7 +687,7 @@ class ProcessorsTest(unittest.TestCase):
self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
self.assertEqual(proc(['hello', 'world']), u'hello world')
- self.assertIsInstance(proc(['hello', 'world']), six.text_type)
+ self.assertIsInstance(proc(['hello', 'world']), str)
def test_compose(self):
proc = Compose(lambda v: v[0], str.upper)
@@ -704,12 +702,12 @@ class ProcessorsTest(unittest.TestCase):
def test_mapcompose(self):
def filter_world(x):
return None if x == 'world' else x
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertRaises(ValueError, proc, [1])
proc = MapCompose(filter_world, lambda x: x + 1)
self.assertRaises(ValueError, proc, 'hello')
@@ -727,11 +725,11 @@ class SelectortemLoaderTest(unittest.TestCase):
</html>
""")
- def test_constructor(self):
+ def test_init_method(self):
l = TestItemLoader()
self.assertEqual(l.selector, None)
- def test_constructor_errors(self):
+ def test_init_method_errors(self):
l = TestItemLoader()
self.assertRaises(RuntimeError, l.add_xpath, 'url', '//a/@href')
self.assertRaises(RuntimeError, l.replace_xpath, 'url', '//a/@href')
@@ -740,7 +738,7 @@ class SelectortemLoaderTest(unittest.TestCase):
self.assertRaises(RuntimeError, l.replace_css, 'name', '#name::text')
self.assertRaises(RuntimeError, l.get_css, '#name::text')
- def test_constructor_with_selector(self):
+ def test_init_method_with_selector(self):
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
@@ -748,7 +746,7 @@ class SelectortemLoaderTest(unittest.TestCase):
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
- def test_constructor_with_selector_css(self):
+ def test_init_method_with_selector_css(self):
sel = Selector(text=u"<html><body><div>marta</div></body></html>")
l = TestItemLoader(selector=sel)
self.assertIs(l.selector, sel)
@@ -756,14 +754,14 @@ class SelectortemLoaderTest(unittest.TestCase):
l.add_css('name', 'div::text')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
- def test_constructor_with_response(self):
+ def test_init_method_with_response(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
l.add_xpath('name', '//div/text()')
self.assertEqual(l.get_output_value('name'), [u'Marta'])
- def test_constructor_with_response_css(self):
+ def test_init_method_with_response_css(self):
l = TestItemLoader(response=self.response)
self.assertTrue(l.selector)
@@ -994,5 +992,53 @@ class SelectJmesTestCase(unittest.TestCase):
)
+# Functions as processors
+
+def function_processor_strip(iterable):
+ return [x.strip() for x in iterable]
+
+
+def function_processor_upper(iterable):
+ return [x.upper() for x in iterable]
+
+
+class FunctionProcessorItem(Item):
+ foo = Field(
+ input_processor=function_processor_strip,
+ output_processor=function_processor_upper,
+ )
+
+
+class FunctionProcessorItemLoader(ItemLoader):
+ default_item_class = FunctionProcessorItem
+
+
+class FunctionProcessorDictLoader(ItemLoader):
+ default_item_class = dict
+ foo_in = function_processor_strip
+ foo_out = function_processor_upper
+
+
+class FunctionProcessorTestCase(unittest.TestCase):
+
+ def test_processor_defined_in_item(self):
+ lo = FunctionProcessorItemLoader()
+ lo.add_value('foo', ' bar ')
+ lo.add_value('foo', [' asdf ', ' qwerty '])
+ self.assertEqual(
+ dict(lo.load_item()),
+ {'foo': ['BAR', 'ASDF', 'QWERTY']}
+ )
+
+ def test_processor_defined_in_item_loader(self):
+ lo = FunctionProcessorDictLoader()
+ lo.add_value('foo', ' bar ')
+ lo.add_value('foo', [' asdf ', ' qwerty '])
+ self.assertEqual(
+ dict(lo.load_item()),
+ {'foo': ['BAR', 'ASDF', 'QWERTY']}
+ )
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index eb9c4a561..7d8c6ec7f 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -3,7 +3,6 @@ import unittest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
-import six
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import DropItem
@@ -23,13 +22,13 @@ class CustomItem(Item):
return "name: %s" % self['name']
-class LoggingContribTest(unittest.TestCase):
+class LogFormatterTestCase(unittest.TestCase):
def setUp(self):
self.formatter = LogFormatter()
self.spider = Spider('default')
- def test_crawled(self):
+ def test_crawled_with_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
@@ -37,6 +36,7 @@ class LoggingContribTest(unittest.TestCase):
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> (referer: None)")
+ def test_crawled_without_referer(self):
req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
res = Response("http://www.example.com", flags=['cached'])
logkws = self.formatter.crawled(req, res, self.spider)
@@ -45,7 +45,7 @@ class LoggingContribTest(unittest.TestCase):
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
- req = Request("http://www.example.com", flags=['test','flag'])
+ req = Request("http://www.example.com", flags=['test', 'flag'])
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
@@ -59,9 +59,19 @@ class LoggingContribTest(unittest.TestCase):
logkws = self.formatter.dropped(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
- assert all(isinstance(x, six.text_type) for x in lines)
+ assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
+ def test_error(self):
+ # In practice, the complete traceback is shown by passing the
+ # 'exc_info' argument to the logging function
+ item = {'key': 'value'}
+ exception = Exception()
+ response = Response("http://www.example.com")
+ logkws = self.formatter.error(item, exception, response, self.spider)
+ logline = logkws['msg'] % logkws['args']
+ self.assertEqual(logline, u"'Error processing {'key': 'value'}'")
+
def test_scraped(self):
item = CustomItem()
item['name'] = u'\xa3'
@@ -69,32 +79,52 @@ class LoggingContribTest(unittest.TestCase):
logkws = self.formatter.scraped(item, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
- assert all(isinstance(x, six.text_type) for x in lines)
+ assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
class LogFormatterSubclass(LogFormatter):
def crawled(self, request, response, spider):
- kwargs = super(LogFormatterSubclass, self).crawled(
- request, response, spider)
+ kwargs = super(LogFormatterSubclass, self).crawled(request, response, spider)
CRAWLEDMSG = (
- u"Crawled (%(status)s) %(request)s (referer: "
- u"%(referer)s)%(flags)s"
+ u"Crawled (%(status)s) %(request)s (referer: %(referer)s) %(flags)s"
)
+ log_args = kwargs['args']
+ log_args['flags'] = str(request.flags)
return {
'level': kwargs['level'],
'msg': CRAWLEDMSG,
- 'args': kwargs['args']
+ 'args': log_args,
}
-class LogformatterSubclassTest(LoggingContribTest):
+class LogformatterSubclassTest(LogFormatterTestCase):
def setUp(self):
self.formatter = LogFormatterSubclass()
self.spider = Spider('default')
+ def test_crawled_with_referer(self):
+ req = Request("http://www.example.com")
+ res = Response("http://www.example.com")
+ logkws = self.formatter.crawled(req, res, self.spider)
+ logline = logkws['msg'] % logkws['args']
+ self.assertEqual(logline,
+ "Crawled (200) <GET http://www.example.com> (referer: None) []")
+
+ def test_crawled_without_referer(self):
+ req = Request("http://www.example.com", headers={'referer': 'http://example.com'}, flags=['cached'])
+ res = Response("http://www.example.com")
+ logkws = self.formatter.crawled(req, res, self.spider)
+ logline = logkws['msg'] % logkws['args']
+ self.assertEqual(logline,
+ "Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
+
def test_flags_in_request(self):
- pass
+ req = Request("http://www.example.com", flags=['test', 'flag'])
+ res = Response("http://www.example.com")
+ logkws = self.formatter.crawled(req, res, self.spider)
+ logline = logkws['msg'] % logkws['args']
+ self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
class SkipMessagesLogFormatter(LogFormatter):
@@ -118,6 +148,7 @@ class DropSomeItemsPipeline(object):
else:
self.drop = True
+
class ShowOrSkipMessagesTestCase(TwistedTestCase):
def setUp(self):
self.mockserver = MockServer()
diff --git a/tests/test_mail.py b/tests/test_mail.py
index b139e98d8..ddb0f1e70 100644
--- a/tests/test_mail.py
+++ b/tests/test_mail.py
@@ -6,6 +6,7 @@ from email.charset import Charset
from scrapy.mail import MailSender
+
class MailSenderTest(unittest.TestCase):
def test_send(self):
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index aea0be825..ebf817c7e 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -3,7 +3,7 @@ from twisted.trial import unittest
from scrapy.settings import Settings
from scrapy.exceptions import NotConfigured
from scrapy.middleware import MiddlewareManager
-import six
+
class M1(object):
@@ -16,6 +16,7 @@ class M1(object):
def process(self, response, request, spider):
pass
+
class M2(object):
def open_spider(self, spider):
@@ -26,6 +27,7 @@ class M2(object):
pass
+
class M3(object):
def process(self, response, request, spider):
@@ -55,6 +57,7 @@ class TestMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process'):
self.methods['process'].append(mw.process)
+
class MiddlewareManagerTest(unittest.TestCase):
def test_init(self):
@@ -66,20 +69,12 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())
- if six.PY2:
- self.assertEqual([x.im_class for x in mwman.methods['open_spider']],
- [M1, M2])
- self.assertEqual([x.im_class for x in mwman.methods['close_spider']],
- [M2, M1])
- self.assertEqual([x.im_class for x in mwman.methods['process']],
- [M1, M3])
- else:
- self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
- [M1, M2])
- self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
- [M2, M1])
- self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
- [M1, M3])
+ self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
+ [M1, M2])
+ self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
+ [M2, M1])
+ self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
+ [M1, M3])
def test_enabled(self):
m1, m2, m3 = M1(), M2(), M3()
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index cb8f8da18..52f2b554e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,12 +1,11 @@
import os
import random
import time
-import hashlib
-import warnings
+from io import BytesIO
from tempfile import mkdtemp
from shutil import rmtree
-from six.moves.urllib.parse import urlparse
-from six import BytesIO
+from unittest import mock
+from urllib.parse import urlparse
from twisted.trial import unittest
from twisted.internet import defer
@@ -15,13 +14,10 @@ from scrapy.pipelines.files import FilesPipeline, FSFilesStore, S3FilesStore, GC
from scrapy.item import Item, Field
from scrapy.http import Request, Response
from scrapy.settings import Settings
-from scrapy.utils.python import to_bytes
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete
from scrapy.utils.test import assert_gcs_environ, get_gcs_content_and_delete
from scrapy.utils.boto import is_botocore
-from tests import mock
-
def _mocked_download_func(request, info):
response = request.meta.get('response')
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 4f7265763..7f1cb4a11 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,7 +1,6 @@
import io
import hashlib
import random
-import warnings
from tempfile import mkdtemp
from shutil import rmtree
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 28e39cefa..1fcc5799e 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,7 +1,3 @@
-from __future__ import print_function
-
-import sys
-
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
@@ -144,10 +140,8 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
# The Failure should encapsulate a FileException ...
self.assertEqual(failure.value, file_exc)
- # ... and if we're running on Python 3 ...
- if sys.version_info.major >= 3:
- # ... it should have the returnValue exception set as its context
- self.assertEqual(failure.value.__context__, def_gen_return_exc)
+ # ... and it should have the returnValue exception set as its context
+ self.assertEqual(failure.value.__context__, def_gen_return_exc)
# Let's calculate the request fingerprint and fake some runtime data...
fp = request_fingerprint(request)
@@ -244,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
m = self.pipe._mockcalled
# only once
- self.assertEqual(m[0], 'get_media_requests') # first hook called
+ self.assertEqual(m[0], 'get_media_requests') # first hook called
self.assertEqual(m.count('get_media_requests'), 1)
self.assertEqual(m.count('item_completed'), 1)
- self.assertEqual(m[-1], 'item_completed') # last hook called
+ self.assertEqual(m[-1], 'item_completed') # last hook called
# twice, one per request
self.assertEqual(m.count('media_to_download'), 2)
# one to handle success and other for failure
@@ -258,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
def test_get_media_requests(self):
# returns single Request (without callback)
req = Request('http://url')
- item = dict(requests=req) # pass a single item
+ item = dict(requests=req) # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
assert request_fingerprint(req) in self.info.downloaded
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
new file mode 100644
index 000000000..bc53f5427
--- /dev/null
+++ b/tests/test_pipelines.py
@@ -0,0 +1,71 @@
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+from twisted.trial import unittest
+
+from scrapy import Spider, signals, Request
+from scrapy.utils.test import get_crawler
+
+from tests.mockserver import MockServer
+
+
+class SimplePipeline:
+ def process_item(self, item, spider):
+ item['pipeline_passed'] = True
+ return item
+
+
+class DeferredPipeline:
+ def cb(self, item):
+ item['pipeline_passed'] = True
+ return item
+
+ def process_item(self, item, spider):
+ d = Deferred()
+ d.addCallback(self.cb)
+ d.callback(item)
+ return d
+
+
+class ItemSpider(Spider):
+ name = 'itemspider'
+
+ def start_requests(self):
+ yield Request(self.mockserver.url('/status?n=200'))
+
+ def parse(self, response):
+ return {'field': 42}
+
+
+class PipelineTestCase(unittest.TestCase):
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+
+ def _on_item_scraped(self, item):
+ self.assertIsInstance(item, dict)
+ self.assertTrue(item.get('pipeline_passed'))
+ self.items.append(item)
+
+ def _create_crawler(self, pipeline_class):
+ settings = {
+ 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
+ }
+ crawler = get_crawler(ItemSpider, settings)
+ crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
+ self.items = []
+ return crawler
+
+ @defer.inlineCallbacks
+ def test_simple_pipeline(self):
+ crawler = self._create_crawler(SimplePipeline)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertEqual(len(self.items), 1)
+
+ @defer.inlineCallbacks
+ def test_deferred_pipeline(self):
+ crawler = self._create_crawler(DeferredPipeline)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertEqual(len(self.items), 1)
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index ae1236bcb..188ec68dd 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,38 +1,53 @@
import json
import os
-import time
+import re
+import sys
+from subprocess import Popen, PIPE
+from urllib.parse import urlsplit, urlunsplit
-from six.moves.urllib.parse import urlsplit, urlunsplit
-from threading import Thread
-from libmproxy import controller, proxy
-from netlib import http_auth
+import pytest
from testfixtures import LogCapture
-
from twisted.internet import defer
from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
+
from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
+from scrapy.utils.test import get_crawler
+
from tests.mockserver import MockServer
+from tests.spiders import SimpleSpider, SingleRequestSpider
-class HTTPSProxy(controller.Master, Thread):
+class MitmProxy:
+ auth_user = 'scrapy'
+ auth_pass = 'scrapy'
- def __init__(self):
- password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy')
- authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy")
+ def start(self):
+ from scrapy.utils.test import get_testenv
+ script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+ """
cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
- 'keys', 'mitmproxy-ca.pem')
- server = proxy.ProxyServer(proxy.ProxyConfig(
- authenticator = authenticator,
- cacert = cert_path),
- 0)
- self.server = server
- Thread.__init__(self)
- controller.Master.__init__(self, server)
+ 'keys', 'mitmproxy-ca.pem')
+ self.proc = Popen([sys.executable,
+ '-c', script,
+ '--listen-host', '127.0.0.1',
+ '--listen-port', '0',
+ '--proxyauth', '%s:%s' % (self.auth_user, self.auth_pass),
+ '--certs', cert_path,
+ '--ssl-insecure',
+ ],
+ stdout=PIPE, env=get_testenv())
+ line = self.proc.stdout.readline().decode('utf-8')
+ host_port = re.search(r'listening at http://([^:]+:\d+)', line).group(1)
+ address = 'http://%s:%s@%s' % (self.auth_user, self.auth_pass, host_port)
+ return address
- def http_address(self):
- return 'http://scrapy:scrapy@%s:%d' % self.server.socket.getsockname()
+ def stop(self):
+ self.proc.kill()
+ self.proc.communicate()
def _wrong_credentials(proxy_url):
@@ -40,6 +55,7 @@ def _wrong_credentials(proxy_url):
bad_auth_proxy[1] = bad_auth_proxy[1].replace('scrapy:scrapy@', 'wrong:wronger@')
return urlunsplit(bad_auth_proxy)
+
class ProxyConnectTestCase(TestCase):
def setUp(self):
@@ -47,17 +63,14 @@ class ProxyConnectTestCase(TestCase):
self.mockserver.__enter__()
self._oldenv = os.environ.copy()
- self._proxy = HTTPSProxy()
- self._proxy.start()
-
- # Wait for the proxy to start.
- time.sleep(1.0)
- os.environ['https_proxy'] = self._proxy.http_address()
- os.environ['http_proxy'] = self._proxy.http_address()
+ self._proxy = MitmProxy()
+ proxy_url = self._proxy.start()
+ os.environ['https_proxy'] = proxy_url
+ os.environ['http_proxy'] = proxy_url
def tearDown(self):
self.mockserver.__exit__(None, None, None)
- self._proxy.shutdown()
+ self._proxy.stop()
os.environ = self._oldenv
@defer.inlineCallbacks
@@ -67,15 +80,7 @@ class ProxyConnectTestCase(TestCase):
yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
self._assert_got_response_code(200, l)
- @defer.inlineCallbacks
- def test_https_noconnect(self):
- proxy = os.environ['https_proxy']
- os.environ['https_proxy'] = proxy + '?noconnect'
- crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(200, l)
-
+ @pytest.mark.xfail(reason='Python 3.6+ fails this earlier', condition=sys.version_info.minor >= 6)
@defer.inlineCallbacks
def test_https_connect_tunnel_error(self):
crawler = get_crawler(SimpleSpider)
@@ -100,17 +105,9 @@ class ProxyConnectTestCase(TestCase):
with LogCapture() as l:
yield crawler.crawl(seed=request)
self._assert_got_response_code(200, l)
- echo = json.loads(crawler.spider.meta['responses'][0].body)
+ echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
- @defer.inlineCallbacks
- def test_https_noconnect_auth_error(self):
- os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
- crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(407, l)
-
def _assert_got_response_code(self, code, log):
print(log)
self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index c9943faa8..a5cdc0de0 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,7 +1,6 @@
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
-import six
from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
@@ -161,9 +160,4 @@ class CallbackKeywordArgumentsTestCase(TestCase):
self.assertEqual(exceptions['takes_less'].exc_info[0], TypeError)
self.assertEqual(str(exceptions['takes_less'].exc_info[1]), "parse_takes_less() got an unexpected keyword argument 'number'")
self.assertEqual(exceptions['takes_more'].exc_info[0], TypeError)
- # py2 and py3 messages are different
- exc_message = str(exceptions['takes_more'].exc_info[1])
- if six.PY2:
- self.assertEqual(exc_message, "parse_takes_more() takes exactly 5 arguments (4 given)")
- elif six.PY3:
- self.assertEqual(exc_message, "parse_takes_more() missing 1 required positional argument: 'other'")
+ self.assertEqual(str(exceptions['takes_more'].exc_info[1]), "parse_takes_more() missing 1 required positional argument: 'other'")
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index f89042b3d..d5a3371ab 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -4,6 +4,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.http import Response, TextResponse, XmlResponse, HtmlResponse, Headers
+
class ResponseTypesTest(unittest.TestCase):
def test_from_filename(self):
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 9aaab560a..27d79437b 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,12 +1,11 @@
# coding=utf-8
from twisted.trial import unittest
-from scrapy.utils.python import to_native_str
def reppy_available():
# check if reppy parser is installed
try:
- from reppy.robots import Robots
+ from reppy.robots import Robots # noqa: F401
except ImportError:
return False
return True
@@ -15,19 +14,21 @@ def reppy_available():
def rerp_available():
# check if robotexclusionrulesparser is installed
try:
- from robotexclusionrulesparser import RobotExclusionRulesParser
+ from robotexclusionrulesparser import RobotExclusionRulesParser # noqa: F401
except ImportError:
return False
return True
+
def protego_available():
# check if protego parser is installed
try:
- from protego import Protego
+ from protego import Protego # noqa: F401
except ImportError:
return False
return True
+
class BaseRobotParserTest:
def _setUp(self, parser_cls):
self.parser_cls = parser_cls
diff --git a/tests/test_selector.py b/tests/test_selector.py
index b2565dd78..09c2546fb 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -1,9 +1,9 @@
-import warnings
import weakref
+
from twisted.trial import unittest
+
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
from scrapy.selector import Selector
-from lxml import etree
class SelectorTestCase(unittest.TestCase):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 1dbacbea3..fda44653a 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,17 +1,15 @@
-import six
import unittest
-import warnings
+from unittest import mock
from scrapy.settings import (BaseSettings, Settings, SettingsAttribute,
SETTINGS_PRIORITIES, get_settings_priority)
-from tests import mock
from . import default_settings
class SettingsGlobalFuncsTest(unittest.TestCase):
def test_get_settings_priority(self):
- for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+ for prio_str, prio_num in SETTINGS_PRIORITIES.items():
self.assertEqual(get_settings_priority(prio_str), prio_num)
self.assertEqual(get_settings_priority(99), 99)
@@ -44,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase):
new_dict = {'three': 11, 'four': 21}
attribute.set(new_dict, 10)
self.assertIsInstance(attribute.value, BaseSettings)
- six.assertCountEqual(self, attribute.value, new_dict)
- six.assertCountEqual(self, original_settings, original_dict)
+ self.assertCountEqual(attribute.value, new_dict)
+ self.assertCountEqual(original_settings, original_dict)
new_settings = BaseSettings({'five': 12}, 0)
attribute.set(new_settings, 0) # Insufficient priority
- six.assertCountEqual(self, attribute.value, new_dict)
+ self.assertCountEqual(attribute.value, new_dict)
attribute.set(new_settings, 10)
- six.assertCountEqual(self, attribute.value, new_settings)
+ self.assertCountEqual(attribute.value, new_settings)
def test_repr(self):
self.assertEqual(repr(self.attribute),
@@ -60,9 +58,6 @@ class SettingsAttributeTest(unittest.TestCase):
class BaseSettingsTest(unittest.TestCase):
- if six.PY3:
- assertItemsEqual = unittest.TestCase.assertCountEqual
-
def setUp(self):
self.settings = BaseSettings()
@@ -152,10 +147,10 @@ class BaseSettingsTest(unittest.TestCase):
self.settings.setmodule(
'tests.test_settings.default_settings', 10)
- self.assertItemsEqual(six.iterkeys(self.settings.attributes),
- six.iterkeys(ctrl_attributes))
+ self.assertCountEqual(self.settings.attributes.keys(),
+ ctrl_attributes.keys())
- for key in six.iterkeys(ctrl_attributes):
+ for key in ctrl_attributes.keys():
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
@@ -231,7 +226,7 @@ class BaseSettingsTest(unittest.TestCase):
}
settings = self.settings
settings.attributes = {key: SettingsAttribute(value, 0) for key, value
- in six.iteritems(test_configuration)}
+ in test_configuration.items()}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
@@ -280,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase):
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
s['TEST'].set(2, 200, 'cmdline')
- six.assertCountEqual(self, s.getwithbase('TEST'),
- {1: 1, 2: 200, 3: 30})
- six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+ self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
+ self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertEqual(s.getwithbase('NONEXISTENT'), {})
def test_maxpriority(self):
@@ -343,9 +337,6 @@ class BaseSettingsTest(unittest.TestCase):
class SettingsTest(unittest.TestCase):
- if six.PY3:
- assertItemsEqual = unittest.TestCase.assertCountEqual
-
def setUp(self):
self.settings = Settings()
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 2220b8ffc..317a27076 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,5 +1,6 @@
import gzip
import inspect
+from unittest import mock
import warnings
from io import BytesIO
@@ -14,11 +15,8 @@ from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
CSVFeedSpider, SitemapSpider
from scrapy.linkextractors import LinkExtractor
from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.trackref import object_ref
from scrapy.utils.test import get_crawler
-from tests import mock
-
class SpiderTest(unittest.TestCase):
@@ -42,12 +40,12 @@ class SpiderTest(unittest.TestCase):
self.assertEqual(list(start_requests), [])
def test_spider_args(self):
- """Constructor arguments are assigned to spider attributes"""
+ """``__init__`` method arguments are assigned to spider attributes"""
spider = self.spider_class('example.com', foo='bar')
self.assertEqual(spider.foo, 'bar')
def test_spider_without_name(self):
- """Constructor arguments are assigned to spider attributes"""
+ """``__init__`` method arguments are assigned to spider attributes"""
self.assertRaises(ValueError, self.spider_class)
self.assertRaises(ValueError, self.spider_class, somearg='foo')
@@ -386,6 +384,14 @@ class CrawlSpiderTest(SpiderTest):
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
+ def test_start_url(self):
+ spider = self.spider_class("example.com")
+ spider.start_url = 'https://www.example.com'
+
+ with self.assertRaisesRegex(AttributeError,
+ r'^Crawling could not start.*$'):
+ list(spider.start_requests())
+
class SitemapSpiderTest(SpiderTest):
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 106da798c..d8be6e277 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -109,6 +109,7 @@ class SpiderLoaderTest(unittest.TestCase):
spiders = spider_loader.list()
self.assertEqual(spiders, [])
+
class DuplicateSpiderNameLoaderTest(unittest.TestCase):
def setUp(self):
diff --git a/tests/test_spiderloader/test_spiders/nested/spider4.py b/tests/test_spiderloader/test_spiders/nested/spider4.py
index 35b71870a..dbd1fb123 100644
--- a/tests/test_spiderloader/test_spiders/nested/spider4.py
+++ b/tests/test_spiderloader/test_spiders/nested/spider4.py
@@ -1,5 +1,6 @@
from scrapy.spiders import Spider
+
class Spider4(Spider):
name = "spider4"
allowed_domains = ['spider4.com']
diff --git a/tests/test_spiderloader/test_spiders/spider0.py b/tests/test_spiderloader/test_spiders/spider0.py
index 75a90794e..af679dbd6 100644
--- a/tests/test_spiderloader/test_spiders/spider0.py
+++ b/tests/test_spiderloader/test_spiders/spider0.py
@@ -1,4 +1,5 @@
from scrapy.spiders import Spider
+
class Spider0(Spider):
allowed_domains = ["scrapy1.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider1.py b/tests/test_spiderloader/test_spiders/spider1.py
index 76efddc7f..6b4317a90 100644
--- a/tests/test_spiderloader/test_spiders/spider1.py
+++ b/tests/test_spiderloader/test_spiders/spider1.py
@@ -1,5 +1,6 @@
from scrapy.spiders import Spider
+
class Spider1(Spider):
name = "spider1"
allowed_domains = ["scrapy1.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider2.py b/tests/test_spiderloader/test_spiders/spider2.py
index 0badd8437..352601863 100644
--- a/tests/test_spiderloader/test_spiders/spider2.py
+++ b/tests/test_spiderloader/test_spiders/spider2.py
@@ -1,5 +1,6 @@
from scrapy.spiders import Spider
+
class Spider2(Spider):
name = "spider2"
allowed_domains = ["scrapy2.org", "scrapy3.org"]
diff --git a/tests/test_spiderloader/test_spiders/spider3.py b/tests/test_spiderloader/test_spiders/spider3.py
index d406f2d4f..84998ba35 100644
--- a/tests/test_spiderloader/test_spiders/spider3.py
+++ b/tests/test_spiderloader/test_spiders/spider3.py
@@ -1,5 +1,6 @@
from scrapy.spiders import Spider
+
class Spider3(Spider):
name = "spider3"
allowed_domains = ['spider3.com']
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 832fd3330..55d665e79 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,3 +1,5 @@
+from unittest import mock
+
from twisted.trial.unittest import TestCase
from twisted.python.failure import Failure
@@ -6,7 +8,6 @@ from scrapy.http import Request, Response
from scrapy.exceptions import _InvalidOutput
from scrapy.utils.test import get_crawler
from scrapy.core.spidermw import SpiderMiddlewareManager
-from tests import mock
class SpiderMiddlewareTestCase(TestCase):
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index 7e4af0d4c..992e60be2 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,13 +1,12 @@
from unittest import TestCase
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+import warnings
from scrapy.http import Response, Request
from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
-from scrapy.spidermiddlewares.offsite import URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
from scrapy.utils.test import get_crawler
-import warnings
+
class TestOffsiteMiddleware(TestCase):
@@ -53,6 +52,7 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
out = list(self.mw.process_spider_output(res, reqs, self.spider))
self.assertEqual(out, reqs)
+
class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
def _get_spider(self):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 6f8727a15..5b7b5e7aa 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -6,7 +6,6 @@ from twisted.internet import defer
from scrapy import Spider, Request
from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
-from tests.spiders import MockServerSpider
class LogExceptionMiddleware:
@@ -34,6 +33,7 @@ class RecoverySpider(Spider):
if not response.meta.get('dont_fail'):
raise TabError()
+
class RecoveryMiddleware:
def process_spider_exception(self, response, exception, spider):
spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
@@ -50,6 +50,7 @@ class FailProcessSpiderInputMiddleware:
spider.logger.info('Middleware: will raise IndexError')
raise IndexError()
+
class ProcessSpiderInputSpiderWithoutErrback(Spider):
name = 'ProcessSpiderInputSpiderWithoutErrback'
custom_settings = {
@@ -177,6 +178,7 @@ class GeneratorRecoverMiddleware:
spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
yield {'processed': [method]}
+
class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware):
pass
@@ -247,6 +249,7 @@ class NotGeneratorRecoverMiddleware:
spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
return [{'processed': [method]}]
+
class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
pass
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 21439c20e..7cc17600c 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,8 +1,7 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
import warnings
-from scrapy.exceptions import NotConfigured
from scrapy.http import Response, Request
from scrapy.settings import Settings
from scrapy.spiders import Spider
@@ -349,6 +348,7 @@ class TestSettingsCustomPolicy(TestRefererMiddleware):
]
+
# --- Tests using Request meta dict to set policy
class TestRequestMetaDefault(MixinDefault, TestRefererMiddleware):
req_meta = {'referrer_policy': POLICY_SCRAPY_DEFAULT}
@@ -518,14 +518,17 @@ class TestPolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
+
class TestPolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
+
class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
+
class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
"""
The empty string means "no-referrer-when-downgrade"
@@ -545,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
(301, 'http://scrapytest.org/3'),
(301, 'http://scrapytest.org/4'),
),
- b'http://scrapytest.org/1', # expected initial referer
- b'http://scrapytest.org/1', # expected referer for the redirection request
+ b'http://scrapytest.org/1', # expected initial referer
+ b'http://scrapytest.org/1', # expected referer for the redirection request
),
( 'https://scrapytest.org/1',
'https://scrapytest.org/2',
@@ -606,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/3'),
(301, 'http://scrapytest.org/4'),
),
- None, # expected initial "Referer"
- None, # expected "Referer" for the redirection request
+ None, # expected initial "Referer"
+ None, # expected "Referer" for the redirection request
),
( 'https://scrapytest.org/1',
'https://scrapytest.org/2',
@@ -645,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
- b'http://scrapytest.org/101', # expected initial "Referer"
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b'http://scrapytest.org/101', # expected initial "Referer"
+ b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
@@ -754,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
- b'http://scrapytest.org/101', # expected initial referer
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b'http://scrapytest.org/101', # expected initial referer
+ b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
@@ -824,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
- b'http://scrapytest.org/101', # expected initial referer
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b'http://scrapytest.org/101', # expected initial referer
+ b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
diff --git a/tests/test_squeues.py b/tests/test_squeues.py
index 3ded5c027..d5fcf2f7f 100644
--- a/tests/test_squeues.py
+++ b/tests/test_squeues.py
@@ -7,16 +7,20 @@ from scrapy.http import Request
from scrapy.loader import ItemLoader
from scrapy.selector import Selector
+
class TestItem(Item):
name = Field()
+
def _test_procesor(x):
return x + x
+
class TestLoader(ItemLoader):
default_item_class = TestItem
name_out = staticmethod(_test_procesor)
+
def nonserializable_object_test(self):
q = self.queue()
try:
@@ -35,6 +39,7 @@ def nonserializable_object_test(self):
sel = Selector(text='<html><body><p>some text</p></body></html>')
self.assertRaises(ValueError, q.push, sel)
+
class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
chunksize = 100000
@@ -53,15 +58,19 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
test_nonserializable_object = nonserializable_object_test
+
class ChunkSize1MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 1
+
class ChunkSize2MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 2
+
class ChunkSize3MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 3
+
class ChunkSize4MarshalFifoDiskQueueTest(MarshalFifoDiskQueueTest):
chunksize = 4
@@ -100,15 +109,19 @@ class PickleFifoDiskQueueTest(MarshalFifoDiskQueueTest):
self.assertEqual(r.url, r2.url)
assert r2.meta['request'] is r2
+
class ChunkSize1PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 1
+
class ChunkSize2PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 2
+
class ChunkSize3PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 3
+
class ChunkSize4PickleFifoDiskQueueTest(PickleFifoDiskQueueTest):
chunksize = 4
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 2033dbe07..2bbbb9e2c 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -1,10 +1,6 @@
from datetime import datetime
import unittest
-
-try:
- from unittest import mock
-except ImportError:
- import mock
+from unittest import mock
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 91bbe43bc..fdc5df166 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,12 +1,12 @@
from unittest import TestCase
-import six
+
import scrapy
class ToplevelTestCase(TestCase):
def test_version(self):
- self.assertIs(type(scrapy.__version__), six.text_type)
+ self.assertIs(type(scrapy.__version__), str)
def test_version_info(self):
self.assertIs(type(scrapy.version_info), tuple)
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 22e39821c..bea0cf3e5 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
import unittest
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index 29937c189..02d8ba51e 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -79,7 +79,7 @@ class BuildComponentListTest(unittest.TestCase):
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
d = {'one': {'a': 'a', 'b': 2}}
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
- d = {'one': 'lorem ipsum',}
+ d = {'one': 'lorem ipsum'}
self.assertRaises(ValueError, build_component_list, {}, d, convert=lambda x: x)
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 65782747b..380c41367 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -14,6 +14,7 @@ try:
except ImportError:
ipy = False
+
class UtilsConsoleTestCase(unittest.TestCase):
def test_get_shell_embed_func(self):
@@ -21,7 +22,7 @@ class UtilsConsoleTestCase(unittest.TestCase):
shell = get_shell_embed_func(['invalid'])
self.assertEqual(shell, None)
- shell = get_shell_embed_func(['invalid','python'])
+ shell = get_shell_embed_func(['invalid', 'python'])
self.assertTrue(callable(shell))
self.assertEqual(shell.__name__, '_embed_standard_shell')
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index c5655df7e..50e1bfd5f 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,7 +1,6 @@
import unittest
import warnings
-from six import assertRaisesRegex
from w3lib.http import basic_auth_header
from scrapy import Request
@@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
def test_too_few_arguments_error(self):
- assertRaisesRegex(
- self,
+ self.assertRaisesRegex(
ValueError,
r"too few arguments|the following arguments are required:\s*url",
lambda: curl_to_request_kwargs("curl"),
@@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
# case 2: ignore_unknown_options=False (raise exception):
- assertRaisesRegex(
- self,
+ self.assertRaisesRegex(
ValueError,
"Unrecognized options:.*--bar.*--baz",
lambda: curl_to_request_kwargs(
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 9455172fa..38a25778e 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,13 +1,8 @@
import copy
import unittest
+from collections.abc import Mapping, MutableMapping
-import six
-if six.PY2:
- from collections import Mapping, MutableMapping
-else:
- from collections.abc import Mapping, MutableMapping
-
-from scrapy.utils.datatypes import CaselessDict, SequenceExclude
+from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
__doctests__ = ['scrapy.utils.datatypes']
@@ -197,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase):
self.assertIn(20, d)
self.assertNotIn(15, d)
- def test_six_range(self):
- import six.moves
- seq = six.moves.range(10**3, 10**6)
- d = SequenceExclude(seq)
- self.assertIn(10**2, d)
- self.assertIn(10**7, d)
- self.assertNotIn(10**4, d)
-
def test_range_step(self):
seq = range(10, 20, 3)
d = SequenceExclude(seq)
@@ -242,5 +229,31 @@ class SequenceExcludeTest(unittest.TestCase):
for v in [-3, "test", 1.1]:
self.assertNotIn(v, d)
+
+class LocalCacheTest(unittest.TestCase):
+
+ def test_cache_with_limit(self):
+ cache = LocalCache(limit=2)
+ cache['a'] = 1
+ cache['b'] = 2
+ cache['c'] = 3
+ self.assertEqual(len(cache), 2)
+ self.assertNotIn('a', cache)
+ self.assertIn('b', cache)
+ self.assertIn('c', cache)
+ self.assertEqual(cache['b'], 2)
+ self.assertEqual(cache['c'], 3)
+
+ def test_cache_without_limit(self):
+ maximum = 10**4
+ cache = LocalCache()
+ for x in range(maximum):
+ cache[str(x)] = x
+ self.assertEqual(len(cache), maximum)
+ for x in range(maximum):
+ self.assertIn(str(x), cache)
+ self.assertEqual(cache[str(x)], x)
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 003bb9b02..dfbe71ae2 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,8 +5,6 @@ from twisted.python.failure import Failure
from scrapy.utils.defer import mustbe_deferred, process_chain, \
process_chain_both, process_parallel, iter_errback
-from six.moves import xrange
-
class MustbeDeferredTest(unittest.TestCase):
def test_success_function(self):
@@ -16,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase):
return steps
dfd = mustbe_deferred(_append, 1)
- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
- steps.append(2) # add another value, that should be catched by assertEqual
+ dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
+ steps.append(2) # add another value, that should be catched by assertEqual
return dfd
def test_unfired_deferred(self):
@@ -29,18 +27,27 @@ class MustbeDeferredTest(unittest.TestCase):
return dfd
dfd = mustbe_deferred(_append, 1)
- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
- steps.append(2) # add another value, that should be catched by assertEqual
+ dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
+ steps.append(2) # add another value, that should be catched by assertEqual
return dfd
+
def cb1(value, arg1, arg2):
return "(cb1 %s %s %s)" % (value, arg1, arg2)
+
+
def cb2(value, arg1, arg2):
return defer.succeed("(cb2 %s %s %s)" % (value, arg1, arg2))
+
+
def cb3(value, arg1, arg2):
return "(cb3 %s %s %s)" % (value, arg1, arg2)
+
+
def cb_fail(value, arg1, arg2):
return Failure(TypeError())
+
+
def eb1(failure, arg1, arg2):
return "(eb1 %s %s %s)" % (failure.value.__class__.__name__, arg1, arg2)
@@ -83,7 +90,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
- for x in xrange(10):
+ for x in range(10):
yield x
errors = []
@@ -93,7 +100,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_bad(self):
def iterbad():
- for x in xrange(10):
+ for x in range(10):
if x == 5:
a = 1/0
yield x
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 3e7236fb1..159ef8f25 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,13 +1,11 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import inspect
import unittest
+from unittest import mock
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
-from tests import mock
-
class MyWarning(UserWarning):
pass
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index 2524153ea..2fac3da1f 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -2,6 +2,7 @@ import unittest
from scrapy.utils.http import decode_chunked_transfer
+
class ChunkedTest(unittest.TestCase):
def test_decode_chunked_transfer(self):
@@ -12,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
chunked_body += "8\r\n" + "sequence\r\n"
chunked_body += "0\r\n\r\n"
body = decode_chunked_transfer(chunked_body)
- self.assertEqual(body, \
- "This is the data in the first chunk\r\n" +
- "and this is the second one\r\n" +
- "consequence")
+ self.assertEqual(body,
+ "This is the data in the first chunk\r\n" +
+ "and this is the second one\r\n" +
+ "consequence")
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 4f9f7a370..cf8ad1f23 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,9 +1,10 @@
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
+
class HttpobjUtilsTest(unittest.TestCase):
def test_urlparse_cached(self):
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 2d845697e..9776dfb2a 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
import os
-import six
+
from twisted.trial import unittest
from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
from scrapy.http import XmlResponse, TextResponse, Response
from tests import get_testdata
+
FOOBAR_NL = u"foo\nbar"
@@ -235,6 +236,7 @@ class LxmlXmliterTestCase(XmliterTestCase):
i = self.xmliter(42, 'product')
self.assertRaises(TypeError, next, i)
+
class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
sample_feed_path = os.path.join(sample_feeds_dir, 'feed-sample3.csv')
@@ -255,8 +257,8 @@ class UtilsCsvTestCase(unittest.TestCase):
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys())))
- self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values())))
+ self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
+ self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
def test_csviter_delimiter(self):
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 742e04803..2c23f3616 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import print_function
import sys
import logging
import unittest
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index e109d5343..6f945cd01 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,11 +1,11 @@
import sys
import os
import unittest
+from unittest import mock
from scrapy.item import Item, Field
from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules
-from tests import mock
__doctests__ = ['scrapy.utils.misc']
@@ -74,7 +74,7 @@ class UtilsMiscTestCase(unittest.TestCase):
self.assertEqual(list(arg_to_iter(100)), [100])
self.assertEqual(list(arg_to_iter(l for l in 'abc')), ['a', 'b', 'c'])
self.assertEqual(list(arg_to_iter([1, 2, 3])), [1, 2, 3])
- self.assertEqual(list(arg_to_iter({'a':1})), [{'a': 1}])
+ self.assertEqual(list(arg_to_iter({'a': 1})), [{'a': 1}])
self.assertEqual(list(arg_to_iter(TestItem(name="john"))), [TestItem(name="john")])
def test_create_instance(self):
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e1148354..b79e0ac1c 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,16 +1,17 @@
-import gc
import functools
+import gc
import operator
+import platform
import unittest
from itertools import count
-import platform
-import six
+from warnings import catch_warnings
from scrapy.utils.python import (
memoizemethod_noargs, binary_is_text, equal_attributes,
- WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode,
+ WeakKeyCache, get_func_args, to_bytes, to_unicode,
without_none_values, MutableChain)
+
__doctests__ = ['scrapy.utils.python']
@@ -21,8 +22,12 @@ class MutableChainTest(unittest.TestCase):
m.extend([7, 8])
m.extend([9, 10], (11, 12))
self.assertEqual(next(m), 0)
- self.assertEqual(m.next(), 1)
- self.assertEqual(m.__next__(), 2)
+ self.assertEqual(m.__next__(), 1)
+ with catch_warnings(record=True) as warnings:
+ self.assertEqual(m.next(), 2)
+ self.assertEqual(len(warnings), 1)
+ self.assertIn('scrapy.utils.python.MutableChain.__next__',
+ str(warnings[0].message))
self.assertEqual(list(m), list(range(3, 13)))
@@ -163,33 +168,6 @@ class UtilsPythonTestCase(unittest.TestCase):
gc.collect()
self.assertFalse(len(wk._weakdict))
- @unittest.skipUnless(six.PY2, "deprecated function")
- def test_stringify_dict(self):
- d = {'a': 123, u'b': b'c', u'd': u'e', object(): u'e'}
- d2 = stringify_dict(d, keys_only=False)
- self.assertEqual(d, d2)
- self.assertIsNot(d, d2) # shouldn't modify in place
- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
- @unittest.skipUnless(six.PY2, "deprecated function")
- def test_stringify_dict_tuples(self):
- tuples = [('a', 123), (u'b', 'c'), (u'd', u'e'), (object(), u'e')]
- d = dict(tuples)
- d2 = stringify_dict(tuples, keys_only=False)
- self.assertEqual(d, d2)
- self.assertIsNot(d, d2) # shouldn't modify in place
- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()), d2.keys())
- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.values()))
-
- @unittest.skipUnless(six.PY2, "deprecated function")
- def test_stringify_dict_keys_only(self):
- d = {'a': 123, u'b': 'c', u'd': u'e', object(): u'e'}
- d2 = stringify_dict(d)
- self.assertEqual(d, d2)
- self.assertIsNot(d, d2) # shouldn't modify in place
- self.assertFalse(any(isinstance(x, six.text_type) for x in d2.keys()))
-
def test_get_func_args(self):
def f1(a, b, c):
pass
@@ -227,16 +205,15 @@ class UtilsPythonTestCase(unittest.TestCase):
if platform.python_implementation() == 'CPython':
# TODO: how do we fix this to return the actual argument names?
- self.assertEqual(get_func_args(six.text_type.split), [])
+ self.assertEqual(get_func_args(str.split), [])
self.assertEqual(get_func_args(" ".join), [])
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
else:
- stripself = not six.PY2 # PyPy3 exposes them as methods
self.assertEqual(
- get_func_args(six.text_type.split, stripself), ['sep', 'maxsplit'])
- self.assertEqual(get_func_args(" ".join, stripself), ['list'])
+ get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
+ self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
self.assertEqual(
- get_func_args(operator.itemgetter(2), stripself), ['obj'])
+ get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
def test_without_none_values(self):
diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py
index 11ac56897..06d9c004c 100644
--- a/tests/test_utils_reqser.py
+++ b/tests/test_utils_reqser.py
@@ -1,8 +1,4 @@
-# -*- coding: utf-8 -*-
import unittest
-import sys
-
-import six
from scrapy.http import Request, FormRequest
from scrapy.spiders import Spider
@@ -80,8 +76,6 @@ class RequestSerializationTest(unittest.TestCase):
self._assert_serializes_ok(r, spider=self.spider)
def test_mixin_private_callback_serialization(self):
- if sys.version_info[0] < 3:
- return
r = Request("http://www.example.com",
callback=self.spider._TestSpiderMixin__mixin_callback,
errback=self.spider.handle_error)
@@ -119,9 +113,8 @@ class RequestSerializationTest(unittest.TestCase):
def test_private_name_mangling(self):
self._assert_mangles_to(
self.spider, '_TestSpider__parse_item_private')
- if sys.version_info[0] >= 3:
- self._assert_mangles_to(
- self.spider, '_TestSpiderMixin__mixin_callback')
+ self._assert_mangles_to(
+ self.spider, '_TestSpiderMixin__mixin_callback')
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 625a32048..3e664fc74 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,9 +1,9 @@
-from __future__ import print_function
import unittest
from scrapy.http import Request
from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
request_authenticate, request_httprepr
+
class UtilsRequestTest(unittest.TestCase):
def test_request_fingerprint(self):
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index bea4dade3..6ebf290c0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,12 +1,13 @@
import os
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
get_meta_refresh, get_base_url, response_status_message)
+
__doctests__ = ['scrapy.utils.response']
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 62edd420d..16b7c5c68 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -66,6 +66,7 @@ class SendCatchLogDeferredTest2(SendCatchLogTest):
def _get_result(self, signal, *a, **kw):
return send_catch_log_deferred(signal, *a, **kw)
+
class SendCatchLogTest2(unittest.TestCase):
def test_error_logged_if_deferred_not_supported(self):
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index 716bb44eb..db323ab31 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -2,6 +2,7 @@ import unittest
from scrapy.utils.sitemap import Sitemap, sitemap_urls_from_robots
+
class SitemapTest(unittest.TestCase):
def test_sitemap(self):
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index d9de1ce77..ee7d17062 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -1,20 +1,19 @@
import unittest
+
+from scrapy import Spider
from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
-from scrapy.spiders import CrawlSpider
-
-class MyBaseSpider(CrawlSpider):
- pass # abstract spider
-
-class MySpider1(MyBaseSpider):
+class MySpider1(Spider):
name = 'myspider1'
-class MySpider2(MyBaseSpider):
+
+class MySpider2(Spider):
name = 'myspider2'
+
class UtilsSpidersTestCase(unittest.TestCase):
def test_iterate_spider_output(self):
@@ -32,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase):
it = iter_spider_classes(tests.test_utils_spider)
self.assertEqual(set(it), {MySpider1, MySpider2})
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index c6072fc0d..16e02f919 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,7 +1,8 @@
-import six
import unittest
+from io import StringIO
+from unittest import mock
+
from scrapy.utils import trackref
-from tests import mock
class Foo(trackref.object_ref):
@@ -38,12 +39,12 @@ Live References
Bar 1 oldest: 0s ago
''')
- @mock.patch('sys.stdout', new_callable=six.StringIO)
+ @mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_empty(self, stdout):
trackref.print_live_refs()
self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
- @mock.patch('sys.stdout', new_callable=six.StringIO)
+ @mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_with_objects(self, stdout):
o1 = Foo() # NOQA
trackref.print_live_refs()
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index e6588055c..21e9a056a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,13 +1,10 @@
# -*- coding: utf-8 -*-
import unittest
-import six
-from six.moves.urllib.parse import urlparse
-
from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
- add_http_if_no_scheme, guess_scheme,
- parse_url, strip_url)
+ add_http_if_no_scheme, guess_scheme, strip_url)
+
__doctests__ = ['scrapy.utils.url']
@@ -187,6 +184,7 @@ class AddHttpIfNoScheme(unittest.TestCase):
class GuessSchemeTest(unittest.TestCase):
pass
+
def create_guess_scheme_t(args):
def do_expected(self):
url = guess_scheme(args[0])
@@ -195,6 +193,7 @@ def create_guess_scheme_t(args):
args[0], url, args[1])
return do_expected
+
def create_skipped_scheme_t(args):
def do_expected(self):
raise unittest.SkipTest(args[2])
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a81946490..746367b41 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,7 +3,6 @@ from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
import os
-import six
import shutil
import OpenSSL.SSL
@@ -78,26 +77,6 @@ class ParseUrlTestCase(unittest.TestCase):
to_bytes(x) if not isinstance(x, int) else x for x in test)
self.assertEqual(client._parse(url), test, url)
- def test_externalUnicodeInterference(self):
- """
- L{client._parse} should return C{str} for the scheme, host, and path
- elements of its return tuple, even when passed an URL which has
- previously been passed to L{urlparse} as a C{unicode} string.
- """
- if not six.PY2:
- raise unittest.SkipTest(
- "Applies only to Py2, as urls can be ONLY unicode on Py3")
- badInput = u'http://example.com/path'
- goodInput = badInput.encode('ascii')
- self._parse(badInput) # cache badInput in urlparse_cached
- scheme, netloc, host, port, path = self._parse(goodInput)
- self.assertTrue(isinstance(scheme, str))
- self.assertTrue(isinstance(netloc, str))
- self.assertTrue(isinstance(host, str))
- self.assertTrue(isinstance(path, str))
- self.assertTrue(isinstance(port, int))
-
-
class ScrapyHTTPPageGetterTests(unittest.TestCase):
@@ -318,7 +297,7 @@ class WebClientTestCase(unittest.TestCase):
def cleanup(passthrough):
# Clean up the server which is hanging around not doing
# anything.
- connected = list(six.iterkeys(self.wrapper.protocols))
+ connected = list(self.wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected:
diff --git a/tox.ini b/tox.ini
index 3f20984cc..d97aab3d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,12 +4,12 @@
# and then run "tox" from this directory.
[tox]
-envlist = py35
+envlist = security,flake8,py3
+minversion = 1.7.0
[testenv]
deps =
-ctests/constraints.txt
- -rrequirements-py3.txt
-rtests/requirements-py3.txt
# Extras
botocore>=1.3.23
@@ -21,13 +21,43 @@ passenv =
GCS_TEST_FILE_URI
GCS_PROJECT_ID
commands =
- py.test --cov=scrapy --cov-report= {posargs:docs scrapy tests}
+ py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
-[testenv:py35]
-basepython = python3.5
+[testenv:security]
+basepython = python3
+deps =
+ bandit
+commands =
+ bandit -r -c .bandit.yml {posargs:scrapy}
-[testenv:py35-pinned]
-basepython = python3.5
+[testenv:flake8]
+basepython = python3
+deps =
+ {[testenv]deps}
+ pytest-flake8
+commands =
+ py.test --flake8 {posargs:docs scrapy tests}
+
+[testenv:pylint]
+basepython = python3
+deps =
+ {[testenv]deps}
+ # Optional dependencies
+ boto
+ reppy
+ robotexclusionrulesparser
+ # Test dependencies
+ pylint
+commands =
+ pylint conftest.py docs extras scrapy setup.py tests
+
+[testenv:pypy3]
+basepython = pypy3
+commands =
+ py.test {posargs:--durations=10 docs scrapy tests}
+
+[testenv:pinned]
+basepython = python3
deps =
-ctests/constraints.txt
cryptography==2.0
@@ -48,41 +78,11 @@ deps =
botocore==1.3.23
Pillow==3.4.2
-[testenv:py36]
-basepython = python3.6
-
-[testenv:py37]
-basepython = python3.7
-
-[testenv:py38]
-basepython = python3.8
-
-[testenv:pypy3]
-basepython = pypy3
-commands =
- py.test {posargs:docs scrapy tests}
-
-[testenv:flake8]
-basepython = python3.8
+[testenv:extra-deps]
deps =
{[testenv]deps}
- pytest-flake8
-commands =
- py.test --flake8 {posargs:docs scrapy tests}
-
-[testenv:pylint]
-basepython = python3.8
-deps =
- {[testenv]deps}
- # Optional dependencies
- boto
reppy
robotexclusionrulesparser
- # Test dependencies
- pylint
-
-commands =
- pylint conftest.py docs extras scrapy setup.py tests
[docs]
changedir = docs
@@ -106,10 +106,3 @@ changedir = {[docs]changedir}
deps = {[docs]deps}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:py38-extra-deps]
-basepython = python3.8
-deps =
- {[testenv]deps}
- reppy
- robotexclusionrulesparser