diff --git a/.bandit.yml b/.bandit.yml
index 00554587a..243379b0b 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,13 +1,15 @@
skips:
- B101
- B105
+- B301
- B303
- B306
- B307
- B311
- B320
- B321
-- B402
+- B402 # https://github.com/scrapy/scrapy/issues/4180
+- B403
- B404
- B406
- B410
diff --git a/.readthedocs.yml b/.readthedocs.yml
new file mode 100644
index 000000000..3c1c3e8be
--- /dev/null
+++ b/.readthedocs.yml
@@ -0,0 +1,7 @@
+version: 2
+sphinx:
+ configuration: docs/conf.py
+python:
+ version: 3.8
+ install:
+ - requirements: docs/requirements.txt
diff --git a/.travis.yml b/.travis.yml
index 9f477e860..c870934e1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,10 +12,9 @@ matrix:
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
- python: 3.5
- env: TOXENV=py35
python: 3.5
- - env: TOXENV=py35-pinned
+ - env: TOXENV=pinned
python: 3.5
- env: TOXENV=py36
python: 3.6
@@ -23,10 +22,10 @@ matrix:
python: 3.7
- env: TOXENV=py38
python: 3.8
- - env: TOXENV=py38-extra-deps
+ - env: TOXENV=extra-deps
python: 3.8
- env: TOXENV=docs
- python: 3.6
+ python: 3.8
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
diff --git a/conftest.py b/conftest.py
index d54ce155c..d37c22436 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,12 +1,19 @@
+from pathlib import Path
+
import pytest
+def _py_files(folder):
+ return (str(p) for p in Path(folder).rglob('*.py'))
+
+
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
+ # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+ *_py_files("tests/CrawlerProcess")
]
-
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html
new file mode 100644
index 000000000..71aff8847
--- /dev/null
+++ b/docs/_tests/quotes.html
@@ -0,0 +1,281 @@
+
+
+
+ Quotes to Scrape
+
+
+
+
+
+
+
+
+
“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is our choices, Harry, that show what we truly are, far more than our abilities.”
+
by J.K. Rowling
+ (about)
+
+
+
+
+
+
“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“The person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.”
+
by Jane Austen
+ (about)
+
+
+
+
+
+
“Imperfection is beauty, madness is genius and it's better to be absolutely ridiculous than absolutely boring.”
+
by Marilyn Monroe
+ (about)
+
+
+
+
+
+
“Try not to become a man of success. Rather become a man of value.”
+
by Albert Einstein
+ (about)
+
+
+
+
+
+
“It is better to be hated for what you are than to be loved for what you are not.”
+
by André Gide
+ (about)
+
+
+
+
+
+
“I have not failed. I've just found 10,000 ways that won't work.”
+
by Thomas A. Edison
+ (about)
+
+
+
+
+
+
“A woman is like a tea bag; you never know how strong it is until it's in hot water.”
+
by Eleanor Roosevelt
+ (about)
+
+
+
+
+
+
“A day without sunshine is like, you know, night.”
+
by Steve Martin
+ (about)
+
+
+
+
+
+
+
+
+
+
+
+
` installation:
+To run all tests::
- ``tox --version``
-
-If your version is older than 1.7.0, please update it first:
-
- ``pip install -U tox``
-
-To run all tests go to the root directory of Scrapy source code and run:
-
- ``tox``
+ tox
To run a specific test (say ``tests/test_loader.py``) use:
@@ -229,7 +221,7 @@ You can also specify a comma-separated list of environmets, and use :ref:`tox’
parallel mode ` to run the tests on multiple environments in
parallel::
- tox -e py27,py36 -p auto
+ tox -e py36,py38 -p auto
To pass command-line options to :doc:`pytest `, add them after
``--`` in your call to :doc:`tox `. Using ``--`` overrides the
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 0ed11c4dc..773b92cea 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,3 @@
--r ../requirements-py3.txt
Sphinx>=2.1
sphinx-hoverxref
sphinx-notfound-page
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 1c9315cd8..e67ce55f9 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -39,7 +39,7 @@ Therefore, you should keep in mind the following things:
.. _topics-inspector:
Inspecting a website
-===================================
+====================
By far the most handy feature of the Developer Tools is the `Inspector`
feature, which allows you to inspect the underlying HTML code of
@@ -79,13 +79,23 @@ sections and tags of a webpage, which greatly improves readability. You can
expand and collapse a tag by clicking on the arrow in front of it or by double
clicking directly on the tag. If we expand the ``span`` tag with the ``class=
"text"`` we will see the quote-text we clicked on. The `Inspector` lets you
-copy XPaths to selected elements. Let's try it out: Right-click on the ``span``
-tag, select ``Copy > XPath`` and paste it in the scrapy shell like so::
+copy XPaths to selected elements. Let's try it out.
+
+First open the Scrapy shell at http://quotes.toscrape.com/ in a terminal:
+
+.. code-block:: none
$ scrapy shell "http://quotes.toscrape.com/"
- (...)
- >>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
- ['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”]
+
+Then, back to your web browser, right-click on the ``span`` tag, select
+``Copy > XPath`` and paste it in the Scrapy shell like so:
+
+.. invisible-code-block: python
+
+ response = load_response('http://quotes.toscrape.com/', 'quotes.html')
+
+>>> response.xpath('/html/body/div/div[2]/div[1]/div[1]/span[1]/text()').getall()
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”']
Adding ``text()`` at the end we are able to extract the first quote with this
basic selector. But this XPath is not really that clever. All it does is
@@ -115,10 +125,10 @@ we'll simply select all ``span`` tags with the ``class="text"`` by using
the `has-class-extension`_:
>>> response.xpath('//span[has-class("text")]/text()').getall()
-['"The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
- '“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
- '“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
- ...]
+['“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”',
+'“It is our choices, Harry, that show what we truly are, far more than our abilities.”',
+'“There are only two ways to live your life. One is as though nothing is a miracle. The other is as though everything is a miracle.”',
+...]
And with one simple, cleverer XPath we are able to extract all quotes from
the page. We could have constructed a loop over our first XPath to increase
@@ -159,7 +169,11 @@ The page is quite similar to the basic `quotes.toscrape.com`_-page,
but instead of the above-mentioned ``Next`` button, the page
automatically loads new quotes when you scroll to the bottom. We
could go ahead and try out different XPaths directly, but instead
-we'll check another quite useful command from the scrapy shell::
+we'll check another quite useful command from the scrapy shell:
+
+.. skip: next
+
+.. code-block:: none
$ scrapy shell "quotes.toscrape.com/scroll"
(...)
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index d65a43afd..b0fb14e24 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -414,6 +414,12 @@ Crawling rules
from which the request originated as second argument. It must return a
``Request`` object or ``None`` (to filter out the request).
+ ``errback`` is a callable or a string (in which case a method from the spider
+ object with that name will be used) to be called if any exception is
+ raised while processing a request generated by the rule.
+ It receives a :class:`Twisted Failure `
+ instance as first parameter.
+
CrawlSpider example
~~~~~~~~~~~~~~~~~~~
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 3bef20bf3..da7a0022b 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python
-from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
diff --git a/pytest.ini b/pytest.ini
index 33c34b8e8..7bb759c28 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -8,7 +8,6 @@ addopts =
--ignore=docs/_ext
--ignore=docs/conf.py
--ignore=docs/news.rst
- --ignore=docs/topics/developer-tools.rst
--ignore=docs/topics/dynamic-content.rst
--ignore=docs/topics/items.rst
--ignore=docs/topics/leaks.rst
@@ -26,52 +25,52 @@ flake8-ignore =
scrapy/http/__init__.py F401
# Issues pending a review:
# extras
- extras/qps-bench-server.py E261 E501
- extras/qpsclient.py E501 E261 E501
+ extras/qps-bench-server.py E501
+ extras/qpsclient.py E501 E501
# scrapy/commands
scrapy/commands/__init__.py E128 E501
scrapy/commands/check.py E501
scrapy/commands/crawl.py E501
scrapy/commands/edit.py E501
- scrapy/commands/fetch.py E401 E501 E128 E502 E731
+ scrapy/commands/fetch.py E401 E501 E128 E731
scrapy/commands/genspider.py E128 E501 E502
scrapy/commands/parse.py E128 E501 E731 E226
scrapy/commands/runspider.py E501
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E501 E502
- scrapy/commands/startproject.py E502 E127 E501 E128
+ scrapy/commands/startproject.py E127 E501 E128
scrapy/commands/version.py E501 E128
# scrapy/contracts
scrapy/contracts/__init__.py E501 W504
- scrapy/contracts/default.py E502 E128
+ scrapy/contracts/default.py E128
# scrapy/core
- scrapy/core/engine.py E261 E501 E128 E127 E306 E502
+ scrapy/core/engine.py E501 E128 E127 E306 E502
scrapy/core/scheduler.py E501
- scrapy/core/scraper.py E501 E306 E261 E128 W504
- scrapy/core/spidermw.py E501 E731 E502 E126 E226
+ scrapy/core/scraper.py E501 E306 E128 W504
+ scrapy/core/spidermw.py E501 E731 E126 E226
scrapy/core/downloader/__init__.py E501
scrapy/core/downloader/contextfactory.py E501 E128 E126
scrapy/core/downloader/middleware.py E501 E502
scrapy/core/downloader/tls.py E501 E305 E241
- scrapy/core/downloader/webclient.py E731 E501 E261 E502 E128 E126 E226
+ scrapy/core/downloader/webclient.py E731 E501 E128 E126 E226
scrapy/core/downloader/handlers/__init__.py E501
scrapy/core/downloader/handlers/ftp.py E501 E305 E128 E127
scrapy/core/downloader/handlers/http10.py E501
scrapy/core/downloader/handlers/http11.py E501
- scrapy/core/downloader/handlers/s3.py E501 E502 E128 E126
+ scrapy/core/downloader/handlers/s3.py E501 E128 E126
# scrapy/downloadermiddlewares
scrapy/downloadermiddlewares/ajaxcrawl.py E501 E226
scrapy/downloadermiddlewares/decompression.py E501
scrapy/downloadermiddlewares/defaultheaders.py E501
scrapy/downloadermiddlewares/httpcache.py E501 E126
- scrapy/downloadermiddlewares/httpcompression.py E502 E128
+ scrapy/downloadermiddlewares/httpcompression.py E501 E128
scrapy/downloadermiddlewares/httpproxy.py E501
scrapy/downloadermiddlewares/redirect.py E501 W504
scrapy/downloadermiddlewares/retry.py E501 E126
scrapy/downloadermiddlewares/robotstxt.py E501
scrapy/downloadermiddlewares/stats.py E501
# scrapy/extensions
- scrapy/extensions/closespider.py E501 E502 E128 E123
+ scrapy/extensions/closespider.py E501 E128 E123
scrapy/extensions/corestats.py E501
scrapy/extensions/feedexport.py E128 E501
scrapy/extensions/httpcache.py E128 E501 E303
@@ -85,13 +84,13 @@ flake8-ignore =
scrapy/http/request/__init__.py E501
scrapy/http/request/form.py E501 E123
scrapy/http/request/json_request.py E501
- scrapy/http/response/__init__.py E501 E128 W293 W291
- scrapy/http/response/text.py E501 W293 E128 E124
+ scrapy/http/response/__init__.py E501 E128
+ scrapy/http/response/text.py E501 E128 E124
# scrapy/linkextractors
- scrapy/linkextractors/__init__.py E731 E502 E501 E402
+ scrapy/linkextractors/__init__.py E731 E501 E402
scrapy/linkextractors/lxmlhtml.py E501 E731 E226
# scrapy/loader
- scrapy/loader/__init__.py E501 E502 E128
+ scrapy/loader/__init__.py E501 E128
scrapy/loader/processors.py E501
# scrapy/pipelines
scrapy/pipelines/files.py E116 E501 E266
@@ -102,7 +101,7 @@ flake8-ignore =
scrapy/selector/unified.py E501 E111
# scrapy/settings
scrapy/settings/__init__.py E501
- scrapy/settings/default_settings.py E501 E261 E114 E116 E226
+ scrapy/settings/default_settings.py E501 E114 E116 E226
scrapy/settings/deprecated.py E501
# scrapy/spidermiddlewares
scrapy/spidermiddlewares/httperror.py E501
@@ -112,25 +111,24 @@ flake8-ignore =
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
- scrapy/spiders/feed.py E501 E261
+ scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
scrapy/utils/benchserver.py E501
- scrapy/utils/conf.py E402 E502 E501
- scrapy/utils/console.py E261 E306 E305
+ scrapy/utils/conf.py E402 E501
+ scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
scrapy/utils/deprecate.py E128 E501 E127 E502
- scrapy/utils/engine.py E261
scrapy/utils/gz.py E305 E501 W504
scrapy/utils/http.py F403 E226
scrapy/utils/httpobj.py E501
scrapy/utils/iterators.py E501 E701
scrapy/utils/log.py E128 W503
- scrapy/utils/markup.py F403 W292
+ scrapy/utils/markup.py F403
scrapy/utils/misc.py E501 E226
- scrapy/utils/multipart.py F403 W292
+ scrapy/utils/multipart.py F403
scrapy/utils/project.py E501
scrapy/utils/python.py E501
scrapy/utils/reactor.py E226
@@ -145,18 +143,17 @@ flake8-ignore =
scrapy/utils/url.py E501 F403 E128 F405
# scrapy
scrapy/__init__.py E402 E501
- scrapy/_monkeypatches.py W293
- scrapy/cmdline.py E502 E501
+ scrapy/cmdline.py E501
scrapy/crawler.py E501
scrapy/dupefilters.py E501 E202
scrapy/exceptions.py E501
- scrapy/exporters.py E501 E261 E226
+ scrapy/exporters.py E501 E226
scrapy/interfaces.py E501
scrapy/item.py E501 E128
scrapy/link.py E501
- scrapy/logformatter.py E501 W293
+ scrapy/logformatter.py E501
scrapy/mail.py E402 E128 E501 E502
- scrapy/middleware.py E502 E128 E501
+ scrapy/middleware.py E128 E501
scrapy/pqueues.py E501
scrapy/responsetypes.py E128 E501 E305
scrapy/robotstxt.py E501
@@ -171,15 +168,15 @@ flake8-ignore =
tests/pipelines.py F841 E226
tests/spiders.py E501 E127
tests/test_closespider.py E501 E127
- tests/test_command_fetch.py E501 E261
+ tests/test_command_fetch.py E501
tests/test_command_parse.py E501 E128 E303 E226
tests/test_command_shell.py E501 E128
tests/test_commands.py E128 E501
- tests/test_contracts.py E501 E128 W293
+ tests/test_contracts.py E501 E128
tests/test_crawl.py E501 E741 E265
tests/test_crawler.py F841 E306 E501
tests/test_dependencies.py F841 E501 E305
- tests/test_downloader_handlers.py E124 E127 E128 E225 E261 E265 E501 E502 E701 E126 E226 E123
+ tests/test_downloader_handlers.py E124 E127 E128 E225 E265 E501 E701 E126 E226 E123
tests/test_downloadermiddleware.py E501
tests/test_downloadermiddleware_ajaxcrawlable.py E501
tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E303 E265 E126
@@ -190,18 +187,18 @@ flake8-ignore =
tests/test_downloadermiddleware_httpcompression.py E501 E251 E126 E123
tests/test_downloadermiddleware_httpproxy.py E501 E128
tests/test_downloadermiddleware_redirect.py E501 E303 E128 E306 E127 E305
- tests/test_downloadermiddleware_retry.py E501 E128 W293 E251 E502 E303 E126
+ tests/test_downloadermiddleware_retry.py E501 E128 E251 E303 E126
tests/test_downloadermiddleware_robotstxt.py E501
tests/test_downloadermiddleware_stats.py E501
- tests/test_dupefilters.py E221 E501 E741 W293 W291 E128 E124
- tests/test_engine.py E401 E501 E502 E128 E261
+ tests/test_dupefilters.py E221 E501 E741 E128 E124
+ tests/test_engine.py E401 E501 E128
tests/test_exporters.py E501 E731 E306 E128 E124
tests/test_extension_telnet.py F841
tests/test_feedexport.py E501 F841 E241
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
- tests/test_http_request.py E402 E501 E261 E127 E128 W293 E502 E128 E502 E126 E123
- tests/test_http_response.py E501 E301 E502 E128 E265
+ tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
+ tests/test_http_response.py E501 E301 E128 E265
tests/test_item.py E701 E128 F841 E306
tests/test_link.py E501
tests/test_linkextractors.py E501 E128 E124
@@ -210,28 +207,28 @@ flake8-ignore =
tests/test_mail.py E128 E501 E305
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E131 E501 E128 E126
- tests/test_pipeline_files.py E501 W293 E303 E272 E226
+ tests/test_pipeline_files.py E501 E303 E272 E226
tests/test_pipeline_images.py F841 E501 E303
- tests/test_pipeline_media.py E501 E741 E731 E128 E261 E306 E502
+ tests/test_pipeline_media.py E501 E741 E731 E128 E306 E502
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501 E305
- tests/test_robotstxt_interface.py E501 W291 E501
+ tests/test_robotstxt_interface.py E501 E501
tests/test_scheduler.py E501 E126 E123
tests/test_selector.py E501 E127
tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
- tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
- tests/test_spidermiddleware_output_chain.py E501 W293 E226
- tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
+ tests/test_spidermiddleware_offsite.py E501 E128 E111
+ tests/test_spidermiddleware_output_chain.py E501 E226
+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
- tests/test_utils_defer.py E306 E261 E501 F841 E226
+ tests/test_utils_defer.py E306 E501 F841 E226
tests/test_utils_deprecate.py F841 E306 E501
- tests/test_utils_http.py E501 E502 E128 W504
+ tests/test_utils_http.py E501 E128 W504
tests/test_utils_iterators.py E501 E128 E129 E303 E241
tests/test_utils_log.py E741 E226
tests/test_utils_python.py E501 E303 E731 E701 E305
@@ -240,11 +237,11 @@ flake8-ignore =
tests/test_utils_response.py E501
tests/test_utils_signal.py E741 F841 E731 E226
tests/test_utils_sitemap.py E128 E501 E124
- tests/test_utils_spider.py E261 E305
+ tests/test_utils_spider.py E305
tests/test_utils_template.py E305
tests/test_utils_url.py E501 E127 E305 E211 E125 E501 E226 E241 E126 E123
tests/test_webclient.py E501 E128 E122 E303 E402 E306 E226 E241 E123 E126
- tests/test_cmdline/__init__.py E502 E501
+ tests/test_cmdline/__init__.py E501
tests/test_settings/__init__.py E501 E128
tests/test_spiderloader/__init__.py E128 E501
tests/test_utils_misc/__init__.py E501
diff --git a/requirements-py3.txt b/requirements-py3.txt
deleted file mode 100644
index 2c98e6f6d..000000000
--- a/requirements-py3.txt
+++ /dev/null
@@ -1,18 +0,0 @@
-parsel>=1.5.0
-PyDispatcher>=2.0.5
-Twisted>=17.9.0
-w3lib>=1.17.0
-protego>=0.1.15
-
-pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
-queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
-cryptography>=2.0 # Earlier versions would fail to install
-
-# Reference versions taken from
-# https://packages.ubuntu.com/xenial/python/
-# https://packages.ubuntu.com/xenial/zope/
-cssselect>=0.9.1
-lxml>=3.5.0
-service_identity>=16.0.0
-six>=1.10.0
-zope.interface>=4.1.3
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 1f8067b35..f74f89bda 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves import copyreg
+import copyreg
# Undo what Twisted's perspective broker adds to pickle register
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 418dc1ac9..ec78f7c91 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import sys
import os
import optparse
@@ -68,7 +67,7 @@ def _pop_command_name(argv):
def _print_header(settings, inproject):
if inproject:
- print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
+ print("Scrapy %s - project: %s\n" % (scrapy.__version__,
settings['BOT_NAME']))
else:
print("Scrapy %s - no active project\n" % scrapy.__version__)
@@ -124,7 +123,7 @@ def execute(argv=None, settings=None):
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)
- parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \
+ parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(),
conflict_handler='resolve')
if not cmdname:
_print_commands(settings, inproject)
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 90c8d56a2..7bbe362e7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,8 +1,7 @@
import sys
import time
import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
import scrapy
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 724b4a1c4..0e149941d 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import sys
from w3lib.url import is_url
@@ -25,12 +24,11 @@ class Command(ScrapyCommand):
def add_options(self, parser):
ScrapyCommand.add_options(self, parser)
- parser.add_option("--spider", dest="spider",
- help="use this spider")
- parser.add_option("--headers", dest="headers", action="store_true", \
- help="print response HTTP headers instead of body")
- parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
- default=False, help="do not handle HTTP 3xx status codes and print response as-is")
+ parser.add_option("--spider", dest="spider", help="use this spider")
+ parser.add_option("--headers", dest="headers", action="store_true",
+ help="print response HTTP headers instead of body")
+ parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
+ default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def _print_headers(self, headers, prefix):
for key, values in headers.items():
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d5498bb5c..adb01fa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import os
import shutil
import string
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 422183ac1..54d7bb228 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ef8acd29c..ff6f1d8cd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import json
import logging
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index ffe3aa2eb..603bafb9f 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import json
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 3b9f6eabb..b123e5c84 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import re
import os
import string
@@ -44,8 +43,8 @@ class Command(ScrapyCommand):
return False
if not re.search(r'^[_a-zA-Z]\w*$', project_name):
- print('Error: Project names must begin with a letter and contain'\
- ' only\nletters, numbers and underscores')
+ print('Error: Project names must begin with a letter and contain'
+ ' only\nletters, numbers and underscores')
elif _module_exists(project_name):
print('Error: Module %r already exists' % project_name)
else:
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 8651948f7..1516c5997 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index e0d425874..3002fc702 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -86,8 +86,8 @@ class ReturnsContract(Contract):
else:
expected = '%s..%s' % (self.min_bound, self.max_bound)
- raise ContractFail("Returned %s %s, expected %s" % \
- (occurrences, self.obj_name, expected))
+ raise ContractFail("Returned %s %s, expected %s" %
+ (occurrences, self.obj_name, expected))
class ScrapesContract(Contract):
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index c5474a57f..157dc3418 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -3,7 +3,6 @@ from time import time
from datetime import datetime
from collections import deque
-import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
@@ -188,7 +187,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
- for slot in six.itervalues(self.slots):
+ for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..39a0b1f51 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,8 +1,9 @@
"""Download handlers for different schemes"""
import logging
+
from twisted.internet import defer
-import six
+
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
- for scheme, clspath in six.iteritems(handlers):
+ for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 39ed67a1a..aef231e82 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..1212feb79 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -2,10 +2,10 @@
import re
import logging
+import warnings
from io import BytesIO
from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
from zope.interface import implementer
from twisted.internet import defer, reactor, protocol
@@ -16,6 +16,7 @@ from twisted.web.http import _DataLoss, PotentialDataLoss
from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI
from twisted.internet.endpoints import TCP4ClientEndpoint
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
@@ -285,6 +286,12 @@ class ScrapyAgent(object):
scheme = _parse(request.url)[0]
proxyHost = to_unicode(proxyHost)
omitConnectTunnel = b'noconnect' in proxyParams
+ if omitConnectTunnel:
+ warnings.warn("Using HTTPS proxies in the noconnect mode is deprecated. "
+ "If you use Crawlera, it doesn't require this mode anymore, "
+ "so you should update scrapy-crawlera to 1.3.0+ "
+ "and remove '?noconnect' from the Crawlera URL.",
+ ScrapyDeprecationWarning)
if scheme == b'https' and not omitConnectTunnel:
proxyAuth = request.headers.get(b'Proxy-Authorization', None)
proxyConf = (proxyHost, proxyPort, proxyAuth)
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index f4a42ce12..d6fbd54ee 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
@@ -32,8 +32,8 @@ def _get_boto_connection():
class S3DownloadHandler(object):
- def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
- httpdownloadhandler=HTTPDownloadHandler, **kw):
+ def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
+ httpdownloadhandler=HTTPDownloadHandler, **kw):
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 7a6a4dfac..38608a429 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,6 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
-import six
-
from twisted.internet import defer
from scrapy.exceptions import _InvalidOutput
@@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
+ (method.__self__.__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, response=response, spider=spider)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response)))
+ (method.__self__.__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, exception=exception, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response)))
+ (method.__self__.__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3fe13414a..fc796e8bb 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,5 @@
from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
@@ -42,7 +42,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
delimiter = b'\n'
def connectionMade(self):
- self.headers = Headers() # bucket for response headers
+ self.headers = Headers() # bucket for response headers
# Method command
self.sendCommand(self.factory.method, self.factory.path)
@@ -88,9 +88,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
if self.factory.url.startswith(b'https'):
self.transport.stopProducing()
- self.factory.noPage(\
- defer.TimeoutError("Getting %s took longer than %s seconds." % \
- (self.factory.url, self.factory.timeout)))
+ self.factory.noPage(
+ defer.TimeoutError("Getting %s took longer than %s seconds." %
+ (self.factory.url, self.factory.timeout)))
class ScrapyHTTPClientFactory(HTTPClientFactory):
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index fa913e528..829e69993 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -25,7 +25,7 @@ class Slot(object):
def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
self.closing = False
- self.inprogress = set() # requests in progress
+ self.inprogress = set() # requests in progress
self.start_requests = iter(start_requests)
self.close_if_idle = close_if_idle
self.nextcall = nextcall
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index db463f989..b3d585cce 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -123,7 +123,7 @@ class Scraper(object):
callback/errback"""
assert isinstance(response, (Response, Failure))
- dfd = self._scrape2(response, request, spider) # returns spiders processed output
+ dfd = self._scrape2(response, request, spider) # returns spiders processed output
dfd.addErrback(self.handle_spider_error, request, response, spider)
dfd.addCallback(self.handle_spider_output, request, response, spider)
return dfd
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 00cee3ada..097a374bf 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
"""
from itertools import chain, islice
-import six
from twisted.python.failure import Failure
from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
@@ -37,15 +36,15 @@ class SpiderMiddlewareManager(MiddlewareManager):
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f: '%s.%s' % (
- six.get_method_self(f).__class__.__name__,
- six.get_method_function(f).__name__)
+ f.__self__.__class__.__name__,
+ f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
try:
result = method(response=response, spider=spider)
if result is not None:
- raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \
+ raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}'
.format(fname(method), type(result)))
except _InvalidOutput:
raise
@@ -70,7 +69,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
elif result is None:
continue
else:
- raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \
+ raise _InvalidOutput('Middleware {} must return None or an iterable, got {}'
.format(fname(method), type(result)))
return _failure
@@ -104,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if _isiterable(result):
result = evaluate_iterable(result, method_index)
else:
- raise _InvalidOutput('Middleware {} must return an iterable, got {}' \
+ raise _InvalidOutput('Middleware {} must return an iterable, got {}'
.format(fname(method), type(result)))
return chain(result, recovered)
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 19b61dc7e..6c7eb737b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,7 +1,6 @@
-import pprint
-import six
-import signal
import logging
+import pprint
+import signal
import warnings
from twisted.internet import reactor, defer
@@ -22,6 +21,7 @@ from scrapy.utils.log import (
get_scrapy_root_handler, install_scrapy_root_handler)
from scrapy import signals
+
logger = logging.getLogger(__name__)
@@ -206,7 +206,7 @@ class CrawlerRunner(object):
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
- if isinstance(spidercls, six.string_types):
+ if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index ba50793bb..7a140fcad 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import re
import logging
-import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@@ -67,7 +65,7 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r''))
+_ajax_crawlable_re = re.compile(r'')
def _has_ajaxcrawlable_meta(text):
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index aeb7578b8..d8dabdf13 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,13 +1,12 @@
import logging
from collections import defaultdict
-import six
-
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.python import to_unicode
+
logger = logging.getLogger(__name__)
@@ -82,8 +81,10 @@ class CookiesMiddleware(object):
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
- cookie_list = [{'name': k, 'value': v} for k, v in \
- six.iteritems(request.cookies)]
+ cookie_list = [
+ {'name': k, 'value': v}
+ for k, v in request.cookies.items()
+ ]
else:
cookie_list = request.cookies
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index e2d73f347..fcea38ef5 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
-from io import BytesIO
-import zipfile
-import tarfile
import logging
+import tarfile
+import zipfile
+from io import BytesIO
from tempfile import mktemp
-import six
-
from scrapy.responsetypes import responsetypes
+
logger = logging.getLogger(__name__)
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
if not response.body:
return response
- for fmt, func in six.iteritems(self._formats):
+ for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 203dee42d..65b652953 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -37,8 +37,9 @@ class HttpCompressionMiddleware(object):
if content_encoding:
encoding = content_encoding.pop()
decoded_body = self._decode(response.body, encoding.lower())
- respcls = responsetypes.from_args(headers=response.headers, \
- url=response.url, body=decoded_body)
+ respcls = responsetypes.from_args(
+ headers=response.headers, url=response.url, body=decoded_body
+ )
kwargs = dict(cls=respcls, body=decoded_body)
if issubclass(respcls, TextResponse):
# force recalculating the encoding until we make sure the
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 5e4542b6c..814ce78fe 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,8 +1,6 @@
import base64
-from urllib.request import _parse_proxy
-
-from six.moves.urllib.parse import unquote, urlunparse
-from six.moves.urllib.request import getproxies, proxy_bypass
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b73f864dd..77cb5aa94 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
import logging
-from six.moves.urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 4d95eb847..ea6a4cfc3 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import os
import logging
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 3defafd60..5bf131312 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -6,15 +6,14 @@ import csv
import io
import pprint
import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+import pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@@ -60,9 +59,9 @@ class BaseItemExporter(object):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
- field_iter = six.iterkeys(item.fields)
+ field_iter = item.fields.keys()
else:
- field_iter = six.iterkeys(item)
+ field_iter = item.keys()
else:
if include_empty:
field_iter = self.fields_to_export
@@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter):
for value in serialized_value:
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
- elif isinstance(serialized_value, six.text_type):
+ elif isinstance(serialized_value, str):
self.xg.characters(serialized_value)
else:
self.xg.characters(str(serialized_value))
@@ -200,7 +199,7 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
- newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+ newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
)
self.csv_writer = csv.writer(self.stream, **kwargs)
self._headers_not_written = True
@@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
- if isinstance(value, (six.text_type, bytes)):
+ if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
def _serialize_dict(self, value):
- for key, val in six.iteritems(value):
+ for key, val in value.items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 9ccf356ec..afb2ed049 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -54,9 +54,9 @@ class CloseSpider(object):
self.crawler.engine.close_spider(spider, 'closespider_pagecount')
def spider_opened(self, spider):
- self.task = reactor.callLater(self.close_on['timeout'], \
- self.crawler.engine.close_spider, spider, \
- reason='closespider_timeout')
+ self.task = reactor.callLater(self.close_on['timeout'],
+ self.crawler.engine.close_spider, spider,
+ reason='closespider_timeout')
def item_scraped(self, item, spider):
self.counter['itemcount'] += 1
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e2492d506..11a63ced2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,7 @@ import logging
import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
-from six.moves.urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote
from ftplib import FTP
from zope.interface import Interface, implementer
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 11403957c..91850683f 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,14 +1,12 @@
-from __future__ import print_function
-
import gzip
import logging
import os
+import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from time import time
from weakref import WeakKeyDictionary
-from six.moves import cPickle as pickle
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 263d8ce4c..892aa8a86 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
"""
import gc
-import six
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 8ba770ec0..2c8e46914 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,5 +1,5 @@
import os
-from six.moves import cPickle as pickle
+import pickle
from scrapy import signals
from scrapy.exceptions import NotConfigured
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 60a14c6f8..0903fd4f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,7 +1,6 @@
import time
-from six.moves.http_cookiejar import (
- CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index f3b46b994..dcaaeddfa 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,4 +1,3 @@
-import six
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@@ -19,7 +18,7 @@ class Headers(CaselessDict):
"""Normalize values to bytes"""
if value is None:
value = []
- elif isinstance(value, (six.text_type, bytes)):
+ elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
@@ -29,10 +28,10 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
- elif isinstance(x, six.text_type):
+ elif isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
- return six.text_type(x).encode(self.encoding)
+ return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
@@ -68,9 +67,6 @@ class Headers(CaselessDict):
self[key] = lst
def items(self):
- return list(self.iteritems())
-
- def iteritems(self):
return ((k, self.getlist(k)) for k in self.keys())
def values(self):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 76a428199..b5c8e1a9a 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,7 +4,6 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-import six
from w3lib.url import safe_url_string
from scrapy.http.headers import Headers
@@ -60,7 +59,7 @@ class Request(object_ref):
return self._url
def _set_url(self, url):
- if not isinstance(url, six.string_types):
+ if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
s = safe_url_string(url, self.encoding)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b6feede07..af02c8484 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
import lxml.html
from parsel.selector import create_root_node
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + \
- u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+ u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index bd09f7534..811d3ad6b 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index a81404afb..e79ce9acc 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from scrapy.http.request import Request
from scrapy.http.headers import Headers
@@ -113,8 +113,8 @@ class Response(object_ref):
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
not only an absolute URL.
-
- :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
+
+ :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
method which supports selectors in addition to absolute/relative URLs
and Link objects.
"""
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 37f450e54..4f9afde87 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
@@ -31,14 +30,14 @@ class TextResponse(Response):
super(TextResponse, self).__init__(*args, **kwargs)
def _set_url(self, url):
- if isinstance(url, six.text_type):
+ if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
- if isinstance(body, six.text_type):
+ if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
@@ -126,7 +125,7 @@ class TextResponse(Response):
Return a :class:`~.Request` instance to follow a link ``url``.
It accepts the same arguments as ``Request.__init__`` method,
but ``url`` can be not only an absolute URL, but also
-
+
* a relative URL;
* a scrapy.link.Link object (e.g. a link extractor result);
* an attribute Selector (not SelectorList) - e.g.
@@ -134,7 +133,7 @@ class TextResponse(Response):
``response.xpath('//img/@src')[0]``.
* a Selector for ```` or ``|\s.*?>)(.*?)<[/ ]?a>",
re.DOTALL | re.IGNORECASE)
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 8940a4d77..2ba6bca45 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -1,9 +1,8 @@
"""
SGMLParser-based Link extractors
"""
-import six
-from six.moves.urllib.parse import urljoin
import warnings
+from urllib.parse import urljoin
from sgmllib import SGMLParser
from w3lib.url import safe_url_string, canonicalize_url
@@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 5189d7cfa..4e5963e99 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -13,7 +13,7 @@ ERRORMSG = u"'Error processing %(item)s'"
class LogFormatter(object):
"""Class for generating log messages for different actions.
-
+
All methods must return a dictionary listing the parameters ``level``, ``msg``
and ``args`` which are going to be used for constructing the log message when
calling ``logging.log``.
@@ -48,7 +48,7 @@ class LogFormatter(object):
}
}
"""
-
+
def crawled(self, request, response, spider):
"""Logs a message when the crawler finds a webpage."""
request_flags = ' %s' % str(request.flags) if request.flags else ''
diff --git a/scrapy/mail.py b/scrapy/mail.py
index d24de2212..9655b8114 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,21 +3,21 @@ Mail sending helpers
See documentation in docs/topics/email.rst
"""
-from io import BytesIO
import logging
-
-from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-from email.mime.nonmultipart import MIMENonMultipart
from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from io import BytesIO
from twisted.internet import defer, reactor, ssl
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
+
logger = logging.getLogger(__name__)
@@ -73,8 +73,7 @@ class MailSender(object):
part = MIMEBase(*mimetype.split('/'))
part.set_payload(f.read())
Encoders.encode_base64(part)
- part.add_header('Content-Disposition', 'attachment; filename="%s"' \
- % attach_name)
+ part.add_header('Content-Disposition', 'attachment', filename=attach_name)
msg.attach(part)
else:
msg.set_payload(body)
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 1cfd8a782..53fa435bb 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -65,8 +65,8 @@ class MiddlewareManager(object):
return process_chain(self.methods[methodname], obj, *args)
def _process_chain_both(self, cb_methodname, eb_methodname, obj, *args):
- return process_chain_both(self.methods[cb_methodname], \
- self.methods[eb_methodname], obj, *args)
+ return process_chain_both(self.methods[cb_methodname],
+ self.methods[eb_methodname], obj, *args)
def open_spider(self, spider):
return self._process_parallel('open_spider', spider)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8d74c5011..6d55c8980 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
-from io import BytesIO
+import logging
import mimetypes
import os
-import os.path
import time
-import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
from collections import defaultdict
-import six
+from email.utils import parsedate_tz, mktime_tz
+from io import BytesIO
+from urllib.parse import urlparse
from twisted.internet import defer, threads
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
+
logger = logging.getLogger(__name__)
@@ -153,14 +152,14 @@ class S3FilesStore(object):
Bucket=self.bucket,
Key=key_name,
Body=buf,
- Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+ Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
- for metakey, metavalue in six.iteritems(meta):
+ for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
})
extra = {}
- for key, value in six.iteritems(headers):
+ for key, value in headers.items():
try:
kwarg = mapping[key]
except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
- blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+ blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e77cef4ff..e9c6b759c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
import functools
import hashlib
from io import BytesIO
-import six
from PIL import Image
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
image, buf = self.convert_image(orig_image)
yield path, image, buf
- for thumb_id, size in six.iteritems(self.thumbs):
+ for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 95dca9a3f..c174addf9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import functools
import logging
from collections import defaultdict
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index de62276c8..91d309147 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -2,11 +2,9 @@
This module implements a class which returns the appropriate Response class
based on different criteria.
"""
-from __future__ import absolute_import
from mimetypes import MimeTypes
from pkgutil import get_data
from io import StringIO
-import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
@@ -37,7 +35,7 @@ class ResponseTypes(object):
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
- for mimetype, cls in six.iteritems(self.CLASSES):
+ for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index f0f9c59dc..0a9af3a62 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,7 +1,6 @@
import sys
import logging
from abc import ABCMeta, abstractmethod
-from six import with_metaclass
from scrapy.utils.python import to_unicode
@@ -26,7 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
return robotstxt_body
-class RobotParser(with_metaclass(ABCMeta)):
+class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):
@@ -56,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
- from six.moves.urllib_robotparser import RobotFileParser
+ from urllib.robotparser import RobotFileParser
self.spider = spider
robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp = RobotFileParser()
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c871e86e0..b6133619c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,4 +1,3 @@
-import six
import json
import copy
from collections.abc import MutableMapping
@@ -23,7 +22,7 @@ def get_settings_priority(priority):
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
numerical value, or directly returns a given numerical priority.
"""
- if isinstance(priority, six.string_types):
+ if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
@@ -173,7 +172,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or [])
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = value.split(',')
return list(value)
@@ -194,7 +193,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or {})
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = json.loads(value)
return dict(value)
@@ -284,7 +283,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(module, six.string_types):
+ if isinstance(module, str):
module = import_module(module)
for key in dir(module):
if key.isupper():
@@ -313,14 +312,14 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(values, six.string_types):
+ if isinstance(values, str):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, values.getpriority(name))
else:
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority='project'):
@@ -377,7 +376,7 @@ class BaseSettings(MutableMapping):
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
- for k, v in six.iteritems(self)}
+ for k, v in self.items()}
def copy_to_dict(self):
"""
@@ -445,7 +444,7 @@ class Settings(BaseSettings):
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities
- for name, val in six.iteritems(self):
+ for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5c9678c01..1e163e1fb 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -84,8 +84,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
DOWNLOADER_CLIENT_TLS_CIPHERS = 'DEFAULT'
-DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
- # also allowing negotiation
+# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
+DOWNLOADER_CLIENT_TLS_METHOD = 'TLS'
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
DOWNLOADER_MIDDLEWARES = {}
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 80b625633..a649d555f 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -3,8 +3,6 @@
See documentation in docs/topics/shell.rst
"""
-from __future__ import print_function
-
import os
import signal
import warnings
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 9a160f62e..481d97e9a 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
from pydispatch import dispatcher
from scrapy.utils import signal as _signal
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 7478faa78..3beca4060 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
from collections import defaultdict
import traceback
import warnings
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index c76e4d5a2..dce2b3598 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,8 +2,8 @@
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
-from six.moves.urllib.parse import urlparse
import warnings
+from urllib.parse import urlparse
from w3lib.url import safe_url_string
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 03000ce54..a2c364c0e 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst
import copy
import warnings
-import six
-
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
from scrapy.linkextractors import LinkExtractor
@@ -18,14 +16,18 @@ from scrapy.utils.python import get_func_args
from scrapy.utils.spider import iterate_spider_output
-def _identity(request, response):
+def _identity(x):
+ return x
+
+
+def _identity_process_request(request, response):
return request
def _get_method(method, spider):
if callable(method):
return method
- elif isinstance(method, six.string_types):
+ elif isinstance(method, str):
return getattr(spider, method, None)
@@ -34,17 +36,20 @@ _default_link_extractor = LinkExtractor()
class Rule(object):
- def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None):
+ def __init__(self, link_extractor=None, callback=None, cb_kwargs=None, follow=None,
+ process_links=None, process_request=None, errback=None):
self.link_extractor = link_extractor or _default_link_extractor
self.callback = callback
+ self.errback = errback
self.cb_kwargs = cb_kwargs or {}
- self.process_links = process_links
- self.process_request = process_request or _identity
+ self.process_links = process_links or _identity
+ self.process_request = process_request or _identity_process_request
self.process_request_argcount = None
self.follow = follow if follow is not None else not callback
def _compile(self, spider):
self.callback = _get_method(self.callback, spider)
+ self.errback = _get_method(self.errback, spider)
self.process_links = _get_method(self.process_links, spider)
self.process_request = _get_method(self.process_request, spider)
self.process_request_argcount = len(get_func_args(self.process_request))
@@ -78,48 +83,59 @@ class CrawlSpider(Spider):
def process_results(self, response, results):
return results
- def _build_request(self, rule, link):
- r = Request(url=link.url, callback=self._response_downloaded)
- r.meta.update(rule=rule, link_text=link.text)
- return r
+ def _build_request(self, rule_index, link):
+ return Request(
+ url=link.url,
+ callback=self._callback,
+ errback=self._errback,
+ meta=dict(rule=rule_index, link_text=link.text),
+ )
def _requests_to_follow(self, response):
if not isinstance(response, HtmlResponse):
return
seen = set()
- for n, rule in enumerate(self._rules):
+ for rule_index, rule in enumerate(self._rules):
links = [lnk for lnk in rule.link_extractor.extract_links(response)
if lnk not in seen]
- if links and rule.process_links:
- links = rule.process_links(links)
- for link in links:
+ for link in rule.process_links(links):
seen.add(link)
- request = self._build_request(n, link)
+ request = self._build_request(rule_index, link)
yield rule._process_request(request, response)
- def _response_downloaded(self, response):
+ def _callback(self, response):
rule = self._rules[response.meta['rule']]
return self._parse_response(response, rule.callback, rule.cb_kwargs, rule.follow)
+ def _errback(self, failure):
+ rule = self._rules[failure.request.meta['rule']]
+ return self._handle_failure(failure, rule.errback)
+
def _parse_response(self, response, callback, cb_kwargs, follow=True):
if callback:
cb_res = callback(response, **cb_kwargs) or ()
cb_res = self.process_results(response, cb_res)
- for requests_or_item in iterate_spider_output(cb_res):
- yield requests_or_item
+ for request_or_item in iterate_spider_output(cb_res):
+ yield request_or_item
if follow and self._follow_links:
for request_or_item in self._requests_to_follow(response):
yield request_or_item
+ def _handle_failure(self, failure, errback):
+ if errback:
+ results = errback(failure) or ()
+ for request_or_item in iterate_spider_output(results):
+ yield request_or_item
+
def _compile_rules(self):
- self._rules = [copy.copy(r) for r in self.rules]
- for rule in self._rules:
- rule._compile(self)
+ self._rules = []
+ for rule in self.rules:
+ self._rules.append(copy.copy(rule))
+ self._rules[-1]._compile(self)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super(CrawlSpider, cls).from_crawler(crawler, *args, **kwargs)
- spider._follow_links = crawler.settings.getbool(
- 'CRAWLSPIDER_FOLLOW_LINKS', True)
+ spider._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)
return spider
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 197812a26..c566f0236 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -100,8 +100,8 @@ class CSVFeedSpider(Spider):
and the file's headers.
"""
- delimiter = None # When this is None, python's csv module's default delimiter is used
- quotechar = None # When this is None, python's csv module's default quotechar is used
+ delimiter = None # When this is None, python's csv module's default delimiter is used
+ quotechar = None # When this is None, python's csv module's default quotechar is used
headers = None
def process_results(self, response, results):
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 534c45c70..d368c7108 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,6 +1,5 @@
import re
import logging
-import six
from scrapy.spiders import Spider
from scrapy.http import Request, XmlResponse
@@ -22,7 +21,7 @@ class SitemapSpider(Spider):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
- if isinstance(c, six.string_types):
+ if isinstance(c, str):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
@@ -86,7 +85,7 @@ class SitemapSpider(Spider):
def regex(x):
- if isinstance(x, six.string_types):
+ if isinstance(x, str):
return re.compile(x)
return x
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 30cc926e5..d5d3be67e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,7 +3,7 @@ Scheduler queues
"""
import marshal
-from six.moves import cPickle as pickle
+import pickle
from queuelib import queue
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 5bbda6e27..cdbe21942 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -1,5 +1,6 @@
import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index b76d5e56e..12321caa5 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,5 @@
"""Boto/botocore helpers"""
-from __future__ import absolute_import
-
from scrapy.exceptions import NotConfigured
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 561bb72fc..23306ca28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,11 +1,9 @@
-from configparser import ConfigParser
import os
import sys
import numbers
+from configparser import ConfigParser
from operator import itemgetter
-import six
-
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
- for k, v in six.iteritems(compdict):
+ for k, v in compdict.items():
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
@@ -33,13 +31,13 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
return compbs
else:
_check_components(compdict)
- return {convert(k): v for k, v in six.iteritems(compdict)}
+ return {convert(k): v for k, v in compdict.items()}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
- for name, value in six.iteritems(compdict):
+ for name, value in compdict.items():
if value is not None and not isinstance(value, numbers.Real):
- raise ValueError('Invalid value {} for component {}, please provide ' \
+ raise ValueError('Invalid value {} for component {}, please provide '
'a real number or None instead'.format(value, name))
# BEGIN Backward compatibility for old (base, custom) call signature
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
- return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+ return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
def arglist_to_dict(arglist):
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 688e28c34..7eb40f0ce 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -47,7 +47,7 @@ def _embed_ptpython_shell(namespace={}, banner=''):
def _embed_standard_shell(namespace={}, banner=''):
"""Start a standard python shell"""
import code
- try: # readline module is only available on unix systems
+ try: # readline module is only available on unix systems
import readline
except ImportError:
pass
@@ -72,9 +72,9 @@ def get_shell_embed_func(shells=None, known_shells=None):
"""Return the first acceptable shell-embed function
from a given list of shell names.
"""
- if shells is None: # list, preference order of shells
+ if shells is None: # list, preference order of shells
shells = DEFAULT_PYTHON_SHELLS.keys()
- if known_shells is None: # available embeddable shells
+ if known_shells is None: # available embeddable shells
known_shells = DEFAULT_PYTHON_SHELLS.copy()
for shell in shells:
if shell in known_shells:
@@ -97,5 +97,5 @@ def start_python_console(namespace=None, banner='', shells=None):
shell = get_shell_embed_func(shells)
if shell is not None:
shell(namespace=namespace, banner=banner)
- except SystemExit: # raised when using exit() in python code.interact
+ except SystemExit: # raised when using exit() in python code.interact
pass
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 7fb25a71d..16639356e 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -1,10 +1,9 @@
import argparse
import warnings
from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
-from six.moves.http_cookies import SimpleCookie
-from six.moves.urllib.parse import urlparse
-from six import iteritems
from w3lib.http import basic_auth_header
@@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
name = name.strip()
val = val.strip()
if name.title() == 'Cookie':
- for name, morsel in iteritems(SimpleCookie(val)):
+ for name, morsel in SimpleCookie(val).items():
cookies[name] = morsel.value
else:
headers.append((name, val))
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 39d389fa6..ffd1537c3 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -5,12 +5,10 @@ Python Standard Library.
This module must not depend on any module outside the Standard Library.
"""
-import copy
import collections
-from collections.abc import Mapping
+import copy
import warnings
-
-import six
+from collections.abc import Mapping
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -151,7 +149,7 @@ class MultiValueDict(dict):
self.setlistdefault(key, []).append(value)
except TypeError:
raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
- for key, value in six.iteritems(kwargs):
+ for key, value in kwargs.items():
self.setlistdefault(key, []).append(value)
@@ -238,65 +236,6 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
-class MergeDict(object):
- """
- A simple class for creating new "virtual" dictionaries that actually look
- up values in more than one dictionary, passed in the ``__init__`` method.
-
- If a key appears in more than one of the given dictionaries, only the
- first occurrence will be used.
- """
- def __init__(self, *dicts):
- warnings.warn(
- "scrapy.utils.datatypes.MergeDict is deprecated in favor "
- "of collections.ChainMap (introduced in Python 3.3)",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- self.dicts = dicts
-
- def __getitem__(self, key):
- for dict_ in self.dicts:
- try:
- return dict_[key]
- except KeyError:
- pass
- raise KeyError
-
- def __copy__(self):
- return self.__class__(*self.dicts)
-
- def get(self, key, default=None):
- try:
- return self[key]
- except KeyError:
- return default
-
- def getlist(self, key):
- for dict_ in self.dicts:
- if key in dict_.keys():
- return dict_.getlist(key)
- return []
-
- def items(self):
- item_list = []
- for dict_ in self.dicts:
- item_list.extend(dict_.items())
- return item_list
-
- def has_key(self, key):
- for dict_ in self.dicts:
- if key in dict_:
- return True
- return False
-
- __contains__ = has_key
-
- def copy(self):
- """Returns a copy of this object."""
- return self.__copy__()
-
-
class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 91ebdae11..9735220ef 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,7 +2,6 @@
pprint and pformat wrappers with colorization support
"""
-from __future__ import print_function
import sys
from pprint import pformat as pformat_
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index b2be0a901..c8d4391b1 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,8 +1,7 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
import weakref
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
_urlparse_cache = weakref.WeakKeyDictionary()
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9693ba768..3c0cb68c3 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,13 +1,13 @@
-import re
import csv
-from io import StringIO
import logging
-import six
+import re
+from io import StringIO
from scrapy.http import TextResponse, Response
from scrapy.selector import Selector
from scrapy.utils.python import re_rsearch, to_unicode
+
logger = logging.getLogger(__name__)
@@ -60,7 +60,7 @@ class _StreamReader(object):
self._text, self.encoding = obj.body, obj.encoding
else:
self._text, self.encoding = obj, 'utf-8'
- self._is_unicode = isinstance(self._text, six.text_type)
+ self._is_unicode = isinstance(self._text, str)
def read(self, n=65535):
self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
- expected_types = (Response, six.text_type, six.binary_type)
+ expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
return obj.text
else:
return obj.body.decode('utf-8')
- elif isinstance(obj, six.text_type):
+ elif isinstance(obj, str):
return obj if unicode else obj.encode('utf-8')
else:
return obj.decode('utf-8') if unicode else obj
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index 2455fcc16..9728c542a 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -11,4 +11,4 @@ from w3lib.html import * # noqa: F401
warnings.warn("Module `scrapy.utils.markup` is deprecated. "
"Please import from `w3lib.html` instead.",
- ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file
+ ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index b74f34451..9955fb1e7 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -6,14 +6,13 @@ from contextlib import contextmanager
from importlib import import_module
from pkgutil import iter_modules
-import six
from w3lib.html import replace_entities
from scrapy.utils.python import flatten, to_unicode
from scrapy.item import BaseItem
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
def arg_to_iter(arg):
@@ -83,7 +82,7 @@ def extract_regex(regex, text, encoding='utf-8'):
* if the regex doesn't contain any group the entire regex matching is returned
"""
- if isinstance(regex, six.string_types):
+ if isinstance(regex, str):
regex = re.compile(regex, re.UNICODE)
try:
@@ -92,7 +91,7 @@ def extract_regex(regex, text, encoding='utf-8'):
strings = regex.findall(text) # full regex or numbered groups
strings = flatten(strings)
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
diff --git a/scrapy/utils/multipart.py b/scrapy/utils/multipart.py
index e81f63152..5dcf791b8 100644
--- a/scrapy/utils/multipart.py
+++ b/scrapy/utils/multipart.py
@@ -12,4 +12,4 @@ from w3lib.form import * # noqa: F401
warnings.warn("Module `scrapy.utils.multipart` is deprecated. "
"If you're using `encode_multipart` function, please use "
"`urllib3.filepost.encode_multipart_formdata` instead",
- ScrapyDeprecationWarning, stacklevel=2)
\ No newline at end of file
+ ScrapyDeprecationWarning, stacklevel=2)
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index f87d5a803..7a7aec9be 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,5 +1,3 @@
-
-from __future__ import absolute_import
import signal
from twisted.internet import reactor
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 1cbda141a..f28c2eaa1 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,5 +1,5 @@
import os
-from six.moves import cPickle as pickle
+import pickle
import warnings
from importlib import import_module
@@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
ENVVAR = 'SCRAPY_SETTINGS_MODULE'
DATADIR_CFG_SECTION = 'datadir'
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 138e86d37..8d829c5a5 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,7 +7,6 @@ import re
import inspect
import weakref
import errno
-import six
from functools import partial, wraps
from itertools import chain
import sys
@@ -65,10 +64,10 @@ def is_listlike(x):
True
>>> is_listlike((x for x in range(3)))
True
- >>> is_listlike(six.moves.xrange(5))
+ >>> is_listlike(range(5))
True
"""
- return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+ return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
def unique(list_, key=lambda x: x):
@@ -87,9 +86,9 @@ def unique(list_, key=lambda x: x):
def to_unicode(text, encoding=None, errors='strict'):
"""Return the unicode representation of a bytes object ``text``. If
``text`` is already an unicode object, return it as-is."""
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return text
- if not isinstance(text, (bytes, six.text_type)):
+ if not isinstance(text, (bytes, str)):
raise TypeError('to_unicode must receive a bytes or str '
'object, got %s' % type(text).__name__)
if encoding is None:
@@ -102,7 +101,7 @@ def to_bytes(text, encoding=None, errors='strict'):
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
- if not isinstance(text, six.string_types):
+ if not isinstance(text, str):
raise TypeError('to_bytes must receive a str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
@@ -138,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
yield (text[offset:], offset)
yield (text, 0)
- if isinstance(pattern, six.string_types):
+ if isinstance(pattern, str):
pattern = re.compile(pattern)
for chunk, offset in _chunk_iter():
@@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
return new_method
-_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
@@ -301,10 +300,10 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
dict or a list of tuples, like any dict ``__init__`` method supports.
"""
d = {}
- for k, v in six.iteritems(dict(dct_or_tuples)):
- k = k.encode(encoding) if isinstance(k, six.text_type) else k
+ for k, v in dict(dct_or_tuples).items():
+ k = k.encode(encoding) if isinstance(k, str) else k
if not keys_only:
- v = v.encode(encoding) if isinstance(v, six.text_type) else v
+ v = v.encode(encoding) if isinstance(v, str) else v
d[k] = v
return d
@@ -346,7 +345,7 @@ def without_none_values(iterable):
value ``None`` have been removed.
"""
try:
- return {k: v for k, v in six.iteritems(iterable) if v is not None}
+ return {k: v for k, v in iterable.items() if v is not None}
except AttributeError:
return type(iterable)((v for v in iterable if v is not None))
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 495564ac0..749bbc387 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,8 +1,6 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
-import six
-
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@@ -87,12 +85,12 @@ def _mangle_private_name(obj, func, name):
def _find_method(obj, func):
if obj:
try:
- func_self = six.get_method_self(func)
+ func_self = func.__self__
except AttributeError: # func has no __self__
pass
else:
if func_self is obj:
- name = six.get_method_function(func).__name__
+ name = func.__func__.__name__
if _is_private_method(name):
return _mangle_private_name(obj, func, name)
return name
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 0fce5a2e1..356753ab5 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -3,16 +3,15 @@ This module provides some useful functions for working with
scrapy.http.Request objects
"""
-from __future__ import print_function
import hashlib
import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_unicode
-
from w3lib.url import canonicalize_url
+
from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
_fingerprint_cache = weakref.WeakKeyDictionary()
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 4742b3e13..2f10cf4de 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
+from urllib.parse import urljoin
+
import lxml.etree
-from six.moves.urllib.parse import urljoin
class Sitemap(object):
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index bf4973fbf..4061d1ea3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,11 +1,10 @@
import logging
import inspect
-import six
-
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
+
logger = logging.getLogger(__name__)
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
# singleton in scrapy.spider.spiders
from scrapy.spiders import Spider
- for obj in six.itervalues(vars(module)):
+ for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, Spider) and \
obj.__module__ == module.__name__ and \
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 9754366df..307c25352 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,7 +2,6 @@
This module contains some assorted functions used in tests
"""
-from __future__ import absolute_import
import os
from importlib import import_module
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index f268e91ff..0f15cf60a 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
import sys
import os
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index e50a989b3..6f5c21624 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,5 +1,4 @@
-from __future__ import print_function
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from twisted.internet import reactor
from twisted.web import server, resource, static, util
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index eed14c5a1..4842b95df 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,12 +9,10 @@ and no performance penalty at all when disabled (as object_ref becomes just an
alias to object in that case).
"""
-from __future__ import print_function
import weakref
from time import time
from operator import itemgetter
from collections import defaultdict
-import six
NoneType = type(None)
@@ -37,13 +35,13 @@ def format_live_refs(ignore=NoneType):
"""Return a tabular representation of tracked objects"""
s = "Live References\n\n"
now = time()
- for cls, wdict in sorted(six.iteritems(live_refs),
+ for cls, wdict in sorted(live_refs.items(),
key=lambda x: x[0].__name__):
if not wdict:
continue
if issubclass(cls, ignore):
continue
- oldest = min(six.itervalues(wdict))
+ oldest = min(wdict.values())
s += "%-30s %6d oldest: %ds ago\n" % (
cls.__name__, len(wdict), now - oldest
)
@@ -57,15 +55,15 @@ def print_live_refs(*a, **kw):
def get_oldest(class_name):
"""Get the oldest object for a specific class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
if not wdict:
break
- return min(six.iteritems(wdict), key=itemgetter(1))[0]
+ return min(wdict.items(), key=itemgetter(1))[0]
def iter_all(class_name):
"""Iterate over all objects of the same class by its class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
- return six.iterkeys(wdict)
+ return wdict.keys()
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 2c7b324a1..c9abb12d5 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
"""
import posixpath
import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
# scrapy.utils.url was moved to w3lib.url and import * ensures this
# move doesn't break old code
diff --git a/setup.py b/setup.py
index 8f5f14f0d..85d797f88 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,6 @@ setup(
'pyOpenSSL>=16.2.0',
'queuelib>=1.4.2',
'service_identity>=16.0.0',
- 'six>=1.10.0',
'w3lib>=1.17.0',
'zope.interface>=4.1.3',
'protego>=0.1.15',
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
new file mode 100644
index 000000000..5f6f1ae30
--- /dev/null
+++ b/tests/CrawlerProcess/simple.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 7ebb8bb62..a45277db9 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -3,9 +3,9 @@ import os
import random
import sys
from subprocess import Popen, PIPE
+from urllib.parse import urlencode
from OpenSSL import SSL
-from six.moves.urllib.parse import urlencode
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.web.static import File
@@ -164,6 +164,12 @@ class Drop(Partial):
request.finish()
+class ArbitraryLengthPayloadResource(LeafResource):
+
+ def render(self, request):
+ return request.content.read()
+
+
class Root(Resource):
def __init__(self):
@@ -177,6 +183,7 @@ class Root(Resource):
self.putChild(b"echo", Echo())
self.putChild(b"payload", PayloadResource())
self.putChild(b"xpayload", EncodingResourceWrapper(PayloadResource(), [GzipEncoderFactory()]))
+ self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
try:
from tests import tests_datadir
self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/')))
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index c4bc1f278..e9bf310b9 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,7 @@
# Tests requirements
jmespath
mitmproxy; python_version >= '3.6'
-mitmproxy==3.0.4; python_version < '3.6'
+mitmproxy<4.0.0; python_version < '3.6'
pytest
pytest-cov
pytest-twisted
diff --git a/tests/spiders.py b/tests/spiders.py
index 2487ecc22..39c8da0b6 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -1,14 +1,14 @@
"""
Some spiders used for testing and benchmarking
"""
-
import time
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
-from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.item import Item
from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import Spider
+from scrapy.spiders.crawl import CrawlSpider, Rule
class MockServerSpider(Spider):
@@ -184,3 +184,35 @@ class DuplicateStartRequestsSpider(MockServerSpider):
def parse(self, response):
self.visited += 1
+
+
+class CrawlSpiderWithErrback(MockServerSpider, CrawlSpider):
+ name = 'crawl_spider_with_errback'
+ custom_settings = {
+ 'RETRY_HTTP_CODES': [], # no need to retry
+ }
+ rules = (
+ Rule(LinkExtractor(), callback='callback', errback='errback', follow=True),
+ )
+
+ def start_requests(self):
+ test_body = b"""
+
+ Page title<title></head>
+ <body>
+ <p><a href="/status?n=200">Item 200</a></p> <!-- callback -->
+ <p><a href="/status?n=201">Item 201</a></p> <!-- callback -->
+ <p><a href="/status?n=404">Item 404</a></p> <!-- errback -->
+ <p><a href="/status?n=500">Item 500</a></p> <!-- errback -->
+ <p><a href="/status?n=501">Item 501</a></p> <!-- errback -->
+ </body>
+ </html>
+ """
+ url = self.mockserver.url("/alpayload")
+ yield Request(url, method="POST", body=test_body)
+
+ def callback(self, response):
+ self.logger.info('[callback] status %i', response.status)
+
+ def errback(self, failure):
+ self.logger.info('[errback] status %i', failure.value.response.status)
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 56cfe642a..da99a6be8 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,13 +1,12 @@
-from io import StringIO
import json
import os
import pstats
import shutil
-import six
-from subprocess import Popen, PIPE
import sys
import tempfile
import unittest
+from io import StringIO
+from subprocess import Popen, PIPE
from scrapy.utils.test import get_testenv
@@ -26,17 +25,15 @@ class CmdlineTest(unittest.TestCase):
return comm.decode(encoding)
def test_default_settings(self):
- self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
- 'default')
+ self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'default')
def test_override_settings_using_set_arg(self):
- self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s', 'TEST1=override'), \
- 'override')
+ self.assertEqual(self._execute('settings', '--get', 'TEST1', '-s',
+ 'TEST1=override'), 'override')
def test_override_settings_using_envvar(self):
self.env['SCRAPY_TEST1'] = 'override'
- self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
- 'override')
+ self.assertEqual(self._execute('settings', '--get', 'TEST1'), 'override')
def test_profiling(self):
path = tempfile.mkdtemp()
@@ -65,5 +62,5 @@ class CmdlineTest(unittest.TestCase):
for char in ("'", "<", ">", 'u"'):
settingsstr = settingsstr.replace(char, '"')
settingsdict = json.loads(settingsstr)
- six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
+ self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
self.assertEqual(200, settingsdict[EXT_PATH])
diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py
index 3fa3ed930..9d3c8fe73 100644
--- a/tests/test_command_fetch.py
+++ b/tests/test_command_fetch.py
@@ -29,6 +29,6 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase):
@defer.inlineCallbacks
def test_headers(self):
_, out, _ = yield self.execute([self.url('/text'), '--headers'])
- out = out.replace(b'\r', b'') # required on win32
+ out = out.replace(b'\r', b'') # required on win32
assert b'Server: TwistedWeb' in out, out
assert b'Content-Type: text/plain' in out
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b2e358700..11d41c1fe 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,6 +1,5 @@
from unittest import TextTestResult
-from six import get_unbound_function
from twisted.internet import defer
from twisted.python import failure
from twisted.trial import unittest
@@ -253,7 +252,7 @@ class ContractsManagerTest(unittest.TestCase):
self.assertEqual(len(contracts), 3)
self.assertEqual(frozenset(type(x) for x in contracts),
frozenset([UrlContract, CallbackKeywordArgumentsContract, ReturnsContract]))
-
+
contracts = self.conman.extract_contracts(spider.returns_item_cb_kwargs)
self.assertEqual(len(contracts), 3)
self.assertEqual(frozenset(type(x) for x in contracts),
@@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
with MockServer() as mockserver:
contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
- get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
- get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
+ TestSameUrlSpider.parse_first.__doc__ = contract_doc
+ TestSameUrlSpider.parse_second.__doc__ = contract_doc
crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
yield crawler.crawl()
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..f433fcea6 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -5,12 +5,12 @@ from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
-from scrapy.http import Request
from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
from scrapy.utils.python import to_unicode
-from tests.spiders import FollowAllSpider, DelaySpider, SimpleSpider, \
- BrokenStartRequestsSpider, SingleRequestSpider, DuplicateStartRequestsSpider
from tests.mockserver import MockServer
+from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
+ SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback)
class CrawlTestCase(TestCase):
@@ -30,25 +30,44 @@ class CrawlTestCase(TestCase):
self.assertEqual(len(crawler.spider.urls_visited), 11) # 10 + start_url
@defer.inlineCallbacks
- def test_delay(self):
- # short to long delays
- yield self._test_delay(0.2, False)
- yield self._test_delay(1, False)
- # randoms
- yield self._test_delay(0.2, True)
- yield self._test_delay(1, True)
+ def test_fixed_delay(self):
+ yield self._test_delay(total=3, delay=0.1)
@defer.inlineCallbacks
- def _test_delay(self, delay, randomize):
- settings = {"DOWNLOAD_DELAY": delay, 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
+ def test_randomized_delay(self):
+ yield self._test_delay(total=3, delay=0.1, randomize=True)
+
+ @defer.inlineCallbacks
+ def _test_delay(self, total, delay, randomize=False):
+ crawl_kwargs = dict(
+ maxlatency=delay * 2,
+ mockserver=self.mockserver,
+ total=total,
+ )
+ tolerance = (1 - (0.6 if randomize else 0.2))
+
+ settings = {"DOWNLOAD_DELAY": delay,
+ 'RANDOMIZE_DOWNLOAD_DELAY': randomize}
crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
- yield crawler.crawl(maxlatency=delay * 2, mockserver=self.mockserver)
- t = crawler.spider.times
- totaltime = t[-1] - t[0]
- avgd = totaltime / (len(t) - 1)
- tolerance = 0.6 if randomize else 0.2
- self.assertTrue(avgd > delay * (1 - tolerance),
- "download delay too small: %s" % avgd)
+ yield crawler.crawl(**crawl_kwargs)
+ times = crawler.spider.times
+ total_time = times[-1] - times[0]
+ average = total_time / (len(times) - 1)
+ self.assertTrue(average > delay * tolerance,
+ "download delay too small: %s" % average)
+
+ # Ensure that the same test parameters would cause a failure if no
+ # download delay is set. Otherwise, it means we are using a combination
+ # of ``total`` and ``delay`` values that are too small for the test
+ # code above to have any meaning.
+ settings["DOWNLOAD_DELAY"] = 0
+ crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+ yield crawler.crawl(**crawl_kwargs)
+ times = crawler.spider.times
+ total_time = times[-1] - times[0]
+ average = total_time / (len(times) - 1)
+ self.assertFalse(average > delay / tolerance,
+ "test total or delay values are too small")
@defer.inlineCallbacks
def test_timeout_success(self):
@@ -140,7 +159,7 @@ class CrawlTestCase(TestCase):
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
- from six.moves.urllib.parse import urlencode
+ from urllib.parse import urlencode
query = urlencode({'raw': '''\
HTTP/1.1 200 OK
Server: Apache-Coyote/1.1
@@ -277,3 +296,15 @@ with multiples lines
self._assert_retried(log)
self.assertIn("Got response 200", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawlspider_with_errback(self):
+ self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
+
+ with LogCapture() as log:
+ yield self.runner.join()
+
+ self.assertIn("[callback] status 200", str(log))
+ self.assertIn("[callback] status 201", str(log))
+ self.assertIn("[errback] status 404", str(log))
+ self.assertIn("[errback] status 500", str(log))
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8eb2389e2..e37a2ff0e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,7 @@
import logging
+import os
+import subprocess
+import sys
import warnings
from twisted.internet import defer
@@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
class BaseCrawlerTest(unittest.TestCase):
@@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, True)
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+ def run_script(self, script_name):
+ script_path = os.path.join(self.script_dir, script_name)
+ args = (sys.executable, script_path)
+ p = subprocess.Popen(args, env=get_testenv(),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ return stderr.decode('utf-8')
+
+ def test_simple(self):
+ log = self.run_script('simple.py')
+ self.assertIn('Spider closed (finished)', log)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 60124b93f..ce39f8545 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -33,7 +33,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.python import to_bytes
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from tests.mockserver import MockServer, ssl_context_factory, Echo
from tests.spiders import SingleRequestSpider
@@ -349,7 +349,7 @@ class HttpTestCase(unittest.TestCase):
return self.download_request(request, Spider('foo')).addCallback(_test)
def test_payload(self):
- body = b'1'*100 # PayloadResource requires body length to be 100
+ body = b'1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)
d = self.download_request(request, Spider('foo'))
d.addCallback(lambda r: r.body)
@@ -695,7 +695,9 @@ class HttpProxyTestCase(unittest.TestCase):
http_proxy = '%s?noconnect' % self.getURL('')
request = Request('https://example.com', meta={'proxy': http_proxy})
- return self.download_request(request, Spider('foo')).addCallback(_test)
+ with self.assertWarnsRegex(ScrapyDeprecationWarning,
+ r'Using HTTPS proxies in the noconnect mode is deprecated'):
+ return self.download_request(request, Spider('foo')).addCallback(_test)
def test_download_without_proxy(self):
def _test(response):
@@ -710,6 +712,9 @@ class HttpProxyTestCase(unittest.TestCase):
class Http10ProxyTestCase(HttpProxyTestCase):
download_handler_cls = HTTP10DownloadHandler
+ def test_download_with_proxy_https_noconnect(self):
+ raise unittest.SkipTest('noconnect is not supported in HTTP10DownloadHandler')
+
class Http11ProxyTestCase(HttpProxyTestCase):
download_handler_cls = HTTP11DownloadHandler
@@ -800,8 +805,8 @@ class S3TestCase(unittest.TestCase):
req = Request('s3://johnsmith/photos/puppy.jpg', headers={'Date': date})
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
- self.assertEqual(httpreq.headers['Authorization'], \
- b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
+ self.assertEqual(httpreq.headers['Authorization'],
+ b'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
def test_request_signing2(self):
# puts an object into the johnsmith bucket.
@@ -813,21 +818,22 @@ class S3TestCase(unittest.TestCase):
})
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
- self.assertEqual(httpreq.headers['Authorization'], \
- b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
+ self.assertEqual(httpreq.headers['Authorization'],
+ b'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
def test_request_signing3(self):
# lists the content of the johnsmith bucket.
date = 'Tue, 27 Mar 2007 19:42:41 +0000'
- req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
- method='GET', headers={
- 'User-Agent': 'Mozilla/5.0',
- 'Date': date,
- })
+ req = Request(
+ 's3://johnsmith/?prefix=photos&max-keys=50&marker=puppy',
+ method='GET', headers={
+ 'User-Agent': 'Mozilla/5.0',
+ 'Date': date,
+ })
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
- self.assertEqual(httpreq.headers['Authorization'], \
- b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
+ self.assertEqual(httpreq.headers['Authorization'],
+ b'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
@@ -836,8 +842,8 @@ class S3TestCase(unittest.TestCase):
method='GET', headers={'Date': date})
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
- self.assertEqual(httpreq.headers['Authorization'], \
- b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
+ self.assertEqual(httpreq.headers['Authorization'],
+ b'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
def test_request_signing5(self):
try:
@@ -850,11 +856,11 @@ class S3TestCase(unittest.TestCase):
# deletes an object from the 'johnsmith' bucket using the
# path-style and Date alternative.
date = 'Tue, 27 Mar 2007 21:20:27 +0000'
- req = Request('s3://johnsmith/photos/puppy.jpg', \
- method='DELETE', headers={
- 'Date': date,
- 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
- })
+ req = Request(
+ 's3://johnsmith/photos/puppy.jpg', method='DELETE', headers={
+ 'Date': date,
+ 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
+ })
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
# botocore does not override Date with x-amz-date
@@ -864,25 +870,26 @@ class S3TestCase(unittest.TestCase):
def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.
date = 'Tue, 27 Mar 2007 21:06:08 +0000'
- req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
- method='PUT', headers={
- 'User-Agent': 'curl/7.15.5',
- 'Host': 'static.johnsmith.net:8080',
- 'Date': date,
- 'x-amz-acl': 'public-read',
- 'content-type': 'application/x-download',
- 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
- 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
- 'X-Amz-Meta-FileChecksum': '0x02661779',
- 'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
- 'Content-Disposition': 'attachment; filename=database.dat',
- 'Content-Encoding': 'gzip',
- 'Content-Length': '5913339',
- })
+ req = Request(
+ 's3://static.johnsmith.net:8080/db-backup.dat.gz',
+ method='PUT', headers={
+ 'User-Agent': 'curl/7.15.5',
+ 'Host': 'static.johnsmith.net:8080',
+ 'Date': date,
+ 'x-amz-acl': 'public-read',
+ 'content-type': 'application/x-download',
+ 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
+ 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
+ 'X-Amz-Meta-FileChecksum': '0x02661779',
+ 'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
+ 'Content-Disposition': 'attachment; filename=database.dat',
+ 'Content-Encoding': 'gzip',
+ 'Content-Length': '5913339',
+ })
with self._mocked_date(date):
httpreq = self.download_request(req, self.spider)
- self.assertEqual(httpreq.headers['Authorization'], \
- b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
+ self.assertEqual(httpreq.headers['Authorization'],
+ b'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
def test_request_signing7(self):
# ensure that spaces are quoted properly before signing
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 51b79b6c3..9c989977e 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -124,7 +124,7 @@ class MaxRetryTimesTest(unittest.TestCase):
# SETTINGS: meta(max_retry_times) = 0
meta_max_retry_times = 0
-
+
req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
@@ -137,7 +137,7 @@ class MaxRetryTimesTest(unittest.TestCase):
self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
def test_with_metakey_greater(self):
-
+
# SETINGS: RETRY_TIMES < meta(max_retry_times)
self.mw.max_retry_times = 2
meta_max_retry_times = 3
@@ -149,7 +149,7 @@ class MaxRetryTimesTest(unittest.TestCase):
self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
def test_with_metakey_lesser(self):
-
+
# SETINGS: RETRY_TIMES > meta(max_retry_times)
self.mw.max_retry_times = 5
meta_max_retry_times = 4
@@ -165,14 +165,14 @@ class MaxRetryTimesTest(unittest.TestCase):
# SETTINGS: meta(max_retry_times) = 4
meta_max_retry_times = 4
- req = Request(self.invalid_url, meta= \
- {'max_retry_times': meta_max_retry_times, 'dont_retry': True})
+ req = Request(self.invalid_url, meta={
+ 'max_retry_times': meta_max_retry_times, 'dont_retry': True
+ })
self._test_retry(req, DNSLookupError('foo'), 0)
-
def _test_retry(self, req, exception, max_retry_times):
-
+
for i in range(0, max_retry_times):
req = self.mw.process_exception(req, exception, self.spider)
assert isinstance(req, Request)
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 8266bf35f..a1645ed96 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
-
from unittest import mock
from twisted.internet import reactor, error
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index e4b0bdf83..0546558bc 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -142,12 +142,12 @@ class RFPDupeFilterTest(unittest.TestCase):
r1 = Request('http://scrapytest.org/index.html')
r2 = Request('http://scrapytest.org/index.html')
-
+
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
assert crawler.stats.get_value('dupefilter/filtered') == 2
- l.check_present(('scrapy.dupefilters', 'DEBUG',
+ l.check_present(('scrapy.dupefilters', 'DEBUG',
('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
' - no more duplicates will be shown'
' (see DUPEFILTER_DEBUG to show all duplicates)')))
@@ -169,7 +169,7 @@ class RFPDupeFilterTest(unittest.TestCase):
r2 = Request('http://scrapytest.org/index.html',
headers={'Referer': 'http://scrapytest.org/INDEX.html'}
)
-
+
dupefilter.log(r1, spider)
dupefilter.log(r2, spider)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 30150391a..25dee7c1f 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,9 +10,10 @@ module with the ``runserver`` argument::
python test_engine.py runserver
"""
-from __future__ import print_function
-import sys, os, re
-from six.moves.urllib.parse import urlparse
+import os
+import re
+import sys
+from urllib.parse import urlparse
from twisted.internet import reactor, defer
from twisted.web import server, static, util
@@ -90,8 +91,8 @@ def start_test_site(debug=False):
port = reactor.listenTCP(0, server.Site(r), interface="127.0.0.1")
if debug:
- print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish." \
- % port.getHost().port)
+ print("Test server running at http://localhost:%d/ - hit Ctrl-C to finish."
+ % port.getHost().port)
return port
@@ -178,7 +179,6 @@ class EngineTest(unittest.TestCase):
@defer.inlineCallbacks
def test_crawler(self):
-
for spider in TestSpider, DictItemsSpider:
self.run = CrawlerRun(spider)
yield self.run.run()
@@ -188,11 +188,15 @@ class EngineTest(unittest.TestCase):
self._assert_scraped_items()
self._assert_signals_catched()
+ @defer.inlineCallbacks
+ def test_crawler_dupefilter(self):
self.run = CrawlerRun(TestDupeFilterSpider)
yield self.run.run()
self._assert_scheduled_requests(urls_to_visit=7)
self._assert_dropped_requests()
+ @defer.inlineCallbacks
+ def test_crawler_itemerror(self):
self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
yield self.run.run()
self._assert_items_error()
@@ -270,7 +274,6 @@ class EngineTest(unittest.TestCase):
self.run.signals_catched[signals.spider_opened])
self.assertEqual({'spider': self.run.spider},
self.run.signals_catched[signals.spider_idle])
- self.run.signals_catched[signals.spider_closed].pop('spider_stats', None) # XXX: remove for scrapy 0.17
self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
self.run.signals_catched[signals.spider_closed])
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 0046c5666..5d1f5c182 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,15 +1,13 @@
-from __future__ import absolute_import
import re
import json
import marshal
+import pickle
import tempfile
import unittest
from io import BytesIO
from datetime import datetime
-from six.moves import cPickle as pickle
import lxml.etree
-import six
from scrapy.item import Item, Field
from scrapy.utils.python import to_unicode
@@ -80,7 +78,7 @@ class BaseItemExporterTest(unittest.TestCase):
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
_, name = list(ie._get_serialized_fields(self.i))[0]
- assert isinstance(name, six.text_type)
+ assert isinstance(name, str)
self.assertEqual(name, u'John\xa3')
def test_field_custom_serializer(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ce3c4f059..2ca57c19d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,15 +1,15 @@
-from __future__ import absolute_import
import os
import csv
import json
import warnings
-from io import BytesIO
import tempfile
import shutil
import string
+from io import BytesIO
+from pathlib import Path
from unittest import mock
-from six.moves.urllib.parse import urljoin, urlparse, quote
-from six.moves.urllib.request import pathname2url
+from urllib.parse import urljoin, urlparse, quote
+from urllib.request import pathname2url
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@@ -28,8 +28,6 @@ from scrapy.extensions.feedexport import (
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
from scrapy.utils.python import to_unicode
-from pathlib import Path
-
class FileFeedStorageTest(unittest.TestCase):
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 0a9ed500a..45ddb42ba 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
from scrapy.http import Request, Response
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index c83cf3b66..cf3fc8496 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -86,9 +86,6 @@ class HeadersTest(unittest.TestCase):
self.assertSortedEqual(h.items(),
[(b'X-Forwarded-For', [b'ip1', b'ip2']),
(b'Content-Type', [b'text/html'])])
- self.assertSortedEqual(h.iteritems(),
- [(b'X-Forwarded-For', [b'ip1', b'ip2']),
- (b'Content-Type', [b'text/html'])])
self.assertSortedEqual(h.values(), [b'ip2', b'text/html'])
def test_update(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9df6ff67b..e30417b30 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,12 +1,10 @@
import unittest
import re
import json
-from unittest import mock
-from urllib.parse import unquote_to_bytes
+import xmlrpc.client
import warnings
-
-from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse, parse_qs
+from unittest import mock
+from urllib.parse import parse_qs, unquote_to_bytes, urlparse
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_unicode
@@ -64,7 +62,7 @@ class RequestTest(unittest.TestCase):
# headers must not be unicode
h = Headers({'key1': u'val1', u'key2': 'val2'})
h[u'newkey'] = u'newval'
- for k, v in h.iteritems():
+ for k, v in h.items():
self.assertIsInstance(k, bytes)
for s in v:
self.assertIsInstance(s, bytes)
@@ -151,7 +149,7 @@ class RequestTest(unittest.TestCase):
r2 = self.request_class(url="http://www.example.com/", body=b"")
assert isinstance(r2.body, bytes)
- self.assertEqual(r2.encoding, 'utf-8') # default encoding
+ self.assertEqual(r2.encoding, 'utf-8') # default encoding
r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
assert isinstance(r3.body, bytes)
@@ -624,8 +622,9 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="two" value="3">
<input type="submit" name="clickable2" value="clicked2">
</form>""")
- req = self.request_class.from_response(response, formdata={'two': '2'}, \
- clickdata={'name': 'clickable2'})
+ req = self.request_class.from_response(
+ response, formdata={'two': '2'}, clickdata={'name': 'clickable2'}
+ )
fs = _qs(req)
self.assertEqual(fs[b'clickable2'], [b'clicked2'])
self.assertFalse(b'clickable1' in fs, fs)
@@ -673,8 +672,9 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="one" value="clicked1">
<input type="hidden" name="two" value="clicked2">
</form>""")
- req = self.request_class.from_response(response, \
- clickdata={u'name': u'clickable', u'value': u'clicked2'})
+ req = self.request_class.from_response(
+ response, clickdata={u'name': u'clickable', u'value': u'clicked2'}
+ )
fs = _qs(req)
self.assertEqual(fs[b'clickable'], [b'clicked2'])
self.assertEqual(fs[b'one'], [b'clicked1'])
@@ -688,8 +688,9 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="poundsign" value="\u00a3">
<input type="hidden" name="eurosign" value="\u20ac">
</form>""")
- req = self.request_class.from_response(response, \
- clickdata={u'name': u'price in \u00a3'})
+ req = self.request_class.from_response(
+ response, clickdata={u'name': u'price in \u00a3'}
+ )
fs = _qs(req, to_unicode=True)
self.assertTrue(fs[u'price in \u00a3'])
@@ -702,8 +703,9 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="yensign" value="\u00a5">
</form>""",
encoding='latin1')
- req = self.request_class.from_response(response, \
- clickdata={u'name': u'price in \u00a5'})
+ req = self.request_class.from_response(
+ response, clickdata={u'name': u'price in \u00a5'}
+ )
fs = _qs(req, to_unicode=True, encoding='latin1')
self.assertTrue(fs[u'price in \u00a5'])
@@ -718,8 +720,9 @@ class FormRequestTest(RequestTest):
<input type="hidden" name="field2" value="value2">
</form>
""")
- req = self.request_class.from_response(response, formname='form2', \
- clickdata={u'name': u'clickable'})
+ req = self.request_class.from_response(
+ response, formname='form2', clickdata={u'name': u'clickable'}
+ )
fs = _qs(req)
self.assertEqual(fs[b'clickable'], [b'clicked2'])
self.assertEqual(fs[b'field2'], [b'value2'])
@@ -727,8 +730,9 @@ class FormRequestTest(RequestTest):
def test_from_response_override_clickable(self):
response = _buildresponse('''<form><input type="submit" name="clickme" value="one"> </form>''')
- req = self.request_class.from_response(response, \
- formdata={'clickme': 'two'}, clickdata={'name': 'clickme'})
+ req = self.request_class.from_response(
+ response, formdata={'clickme': 'two'}, clickdata={'name': 'clickme'}
+ )
fs = _qs(req)
self.assertEqual(fs[b'clickme'], [b'two'])
@@ -855,7 +859,7 @@ class FormRequestTest(RequestTest):
<form name="form2" action="post.php" method="POST">
<input type="hidden" name="two" value="2">
</form>""")
- self.assertRaises(IndexError, self.request_class.from_response, \
+ self.assertRaises(IndexError, self.request_class.from_response,
response, formname="form3", formnumber=2)
def test_from_response_formid_exists(self):
@@ -909,7 +913,7 @@ class FormRequestTest(RequestTest):
<form id="form2" name="form2" action="post.php" method="POST">
<input type="hidden" name="two" value="2">
</form>""")
- self.assertRaises(IndexError, self.request_class.from_response, \
+ self.assertRaises(IndexError, self.request_class.from_response,
response, formid="form3", formnumber=2)
def test_from_response_select(self):
@@ -1220,7 +1224,7 @@ class XmlRpcRequestTest(RequestTest):
r = self.request_class('http://scrapytest.org/rpc2', **kwargs)
self.assertEqual(r.headers[b'Content-Type'], b'text/xml')
self.assertEqual(r.body,
- to_bytes(xmlrpclib.dumps(**kwargs),
+ to_bytes(xmlrpc.client.dumps(**kwargs),
encoding=kwargs.get('encoding', 'utf-8')))
self.assertEqual(r.method, 'POST')
self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8'))
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 883c943da..960ecea3e 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
import unittest
-import six
from w3lib.encoding import resolve_encoding
from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
@@ -102,7 +101,7 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(r4.flags, [])
def _assert_response_values(self, response, encoding, body):
- if isinstance(body, six.text_type):
+ if isinstance(body, str):
body_unicode = body
body_bytes = body.encode(encoding)
else:
@@ -110,7 +109,7 @@ class BaseResponseTest(unittest.TestCase):
body_bytes = body
assert isinstance(response.body, bytes)
- assert isinstance(response.text, six.text_type)
+ assert isinstance(response.text, str)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_bytes)
self.assertEqual(response.body_as_unicode(), body_unicode)
@@ -220,11 +219,11 @@ class TextResponseTest(BaseResponseTest):
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
# check body_as_unicode
- self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
+ self.assertTrue(isinstance(r1.body_as_unicode(), str))
self.assertEqual(r1.body_as_unicode(), unicode_string)
# check response.text
- self.assertTrue(isinstance(r1.text, six.text_type))
+ self.assertTrue(isinstance(r1.text, str))
self.assertEqual(r1.text, unicode_string)
def test_encoding(self):
@@ -317,8 +316,8 @@ class TextResponseTest(BaseResponseTest):
assert u'SUFFIX' in r.text, repr(r.text)
# Do not destroy html tags due to encoding bugs
- r = self.response_class("http://example.com", encoding='utf-8', \
- body=b'\xf0<span>value</span>')
+ r = self.response_class("http://example.com", encoding='utf-8',
+ body=b'\xf0<span>value</span>')
assert u'<span>value</span>' in r.text, repr(r.text)
# FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
diff --git a/tests/test_item.py b/tests/test_item.py
index 49117ef04..30463a0f5 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -3,8 +3,6 @@ import unittest
from unittest import mock
from warnings import catch_warnings
-import six
-
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
@@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase):
class ItemMetaClassCellRegression(unittest.TestCase):
def test_item_meta_classcell_regression(self):
- class MyItem(six.with_metaclass(ItemMeta, Item)):
+ class MyItem(Item, metaclass=ItemMeta):
def __init__(self, *args, **kwargs):
# This call to super() trigger the __classcell__ propagation
# requirement. When not done properly raises an error:
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 6bfc31dbf..579a85ff6 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,8 +1,6 @@
from functools import partial
import unittest
-import six
-
from scrapy.http import HtmlResponse
from scrapy.item import Item, Field
from scrapy.loader import ItemLoader
@@ -157,7 +155,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_get_value(self):
il = NameItemLoader()
- self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
+ self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
@@ -258,7 +256,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
- name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
+ name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
il.add_value('name', u'marta')
@@ -266,7 +264,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_extend_default_input_processors(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
- name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
+ name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
il = ChildDefaultedItemLoader()
il.add_value('name', u'marta')
@@ -689,7 +687,7 @@ class ProcessorsTest(unittest.TestCase):
self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
self.assertEqual(proc(['hello', 'world']), u'hello world')
- self.assertIsInstance(proc(['hello', 'world']), six.text_type)
+ self.assertIsInstance(proc(['hello', 'world']), str)
def test_compose(self):
proc = Compose(lambda v: v[0], str.upper)
@@ -704,12 +702,12 @@ class ProcessorsTest(unittest.TestCase):
def test_mapcompose(self):
def filter_world(x):
return None if x == 'world' else x
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertRaises(ValueError, proc, [1])
proc = MapCompose(filter_world, lambda x: x + 1)
self.assertRaises(ValueError, proc, 'hello')
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index d0b23a8c4..7d8c6ec7f 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -3,7 +3,6 @@ import unittest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
-import six
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import DropItem
@@ -60,7 +59,7 @@ class LogFormatterTestCase(unittest.TestCase):
logkws = self.formatter.dropped(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
- assert all(isinstance(x, six.text_type) for x in lines)
+ assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
def test_error(self):
@@ -80,7 +79,7 @@ class LogFormatterTestCase(unittest.TestCase):
logkws = self.formatter.scraped(item, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
- assert all(isinstance(x, six.text_type) for x in lines)
+ assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index bd40e4103..141141671 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,11 +1,11 @@
import os
import random
import time
+from io import BytesIO
from tempfile import mkdtemp
from shutil import rmtree
from unittest import mock
-from six.moves.urllib.parse import urlparse
-from six import BytesIO
+from urllib.parse import urlparse
from twisted.trial import unittest
from twisted.internet import defer
@@ -58,7 +58,6 @@ class FilesPipelineTestCase(unittest.TestCase):
self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
//+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
-
def test_fs_store(self):
assert isinstance(self.pipeline.store, FSFilesStore)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ad958e25f..1fcc5799e 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
@@ -240,10 +238,10 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
self.assertEqual(new_item['results'], [(True, rsp1), (False, fail)])
m = self.pipe._mockcalled
# only once
- self.assertEqual(m[0], 'get_media_requests') # first hook called
+ self.assertEqual(m[0], 'get_media_requests') # first hook called
self.assertEqual(m.count('get_media_requests'), 1)
self.assertEqual(m.count('item_completed'), 1)
- self.assertEqual(m[-1], 'item_completed') # last hook called
+ self.assertEqual(m[-1], 'item_completed') # last hook called
# twice, one per request
self.assertEqual(m.count('media_to_download'), 2)
# one to handle success and other for failure
@@ -254,7 +252,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
def test_get_media_requests(self):
# returns single Request (without callback)
req = Request('http://url')
- item = dict(requests=req) # pass a single item
+ item = dict(requests=req) # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
assert new_item is item
assert request_fingerprint(req) in self.info.downloaded
diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py
new file mode 100644
index 000000000..bc53f5427
--- /dev/null
+++ b/tests/test_pipelines.py
@@ -0,0 +1,71 @@
+from twisted.internet import defer
+from twisted.internet.defer import Deferred
+from twisted.trial import unittest
+
+from scrapy import Spider, signals, Request
+from scrapy.utils.test import get_crawler
+
+from tests.mockserver import MockServer
+
+
+class SimplePipeline:
+ def process_item(self, item, spider):
+ item['pipeline_passed'] = True
+ return item
+
+
+class DeferredPipeline:
+ def cb(self, item):
+ item['pipeline_passed'] = True
+ return item
+
+ def process_item(self, item, spider):
+ d = Deferred()
+ d.addCallback(self.cb)
+ d.callback(item)
+ return d
+
+
+class ItemSpider(Spider):
+ name = 'itemspider'
+
+ def start_requests(self):
+ yield Request(self.mockserver.url('/status?n=200'))
+
+ def parse(self, response):
+ return {'field': 42}
+
+
+class PipelineTestCase(unittest.TestCase):
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+
+ def _on_item_scraped(self, item):
+ self.assertIsInstance(item, dict)
+ self.assertTrue(item.get('pipeline_passed'))
+ self.items.append(item)
+
+ def _create_crawler(self, pipeline_class):
+ settings = {
+ 'ITEM_PIPELINES': {__name__ + '.' + pipeline_class.__name__: 1},
+ }
+ crawler = get_crawler(ItemSpider, settings)
+ crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
+ self.items = []
+ return crawler
+
+ @defer.inlineCallbacks
+ def test_simple_pipeline(self):
+ crawler = self._create_crawler(SimplePipeline)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertEqual(len(self.items), 1)
+
+ @defer.inlineCallbacks
+ def test_deferred_pipeline(self):
+ crawler = self._create_crawler(DeferredPipeline)
+ yield crawler.crawl(mockserver=self.mockserver)
+ self.assertEqual(len(self.items), 1)
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 277455751..188ec68dd 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,20 +1,20 @@
import json
import os
import re
-from subprocess import Popen, PIPE
import sys
+from subprocess import Popen, PIPE
+from urllib.parse import urlsplit, urlunsplit
import pytest
-from six.moves.urllib.parse import urlsplit, urlunsplit
from testfixtures import LogCapture
-
from twisted.internet import defer
from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
+from scrapy.utils.test import get_crawler
+
from tests.mockserver import MockServer
+from tests.spiders import SimpleSpider, SingleRequestSpider
class MitmProxy:
@@ -108,32 +108,6 @@ class ProxyConnectTestCase(TestCase):
echo = json.loads(crawler.spider.meta['responses'][0].text)
self.assertTrue('Proxy-Authorization' not in echo['headers'])
- # The noconnect mode isn't supported by the current mitmproxy, it returns
- # "Invalid request scheme: https" as it doesn't seem to support full URLs in GET at all,
- # and it's not clear what behavior is intended by Scrapy and by mitmproxy here.
- # https://github.com/mitmproxy/mitmproxy/issues/848 may be related.
- # The Scrapy noconnect mode was required, at least in the past, to work with Crawlera,
- # and https://github.com/scrapy-plugins/scrapy-crawlera/pull/44 seems to be related.
-
- @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
- @defer.inlineCallbacks
- def test_https_noconnect(self):
- proxy = os.environ['https_proxy']
- os.environ['https_proxy'] = proxy + '?noconnect'
- crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(200, l)
-
- @pytest.mark.xfail(reason='mitmproxy gives an error for noconnect requests')
- @defer.inlineCallbacks
- def test_https_noconnect_auth_error(self):
- os.environ['https_proxy'] = _wrong_credentials(os.environ['https_proxy']) + '?noconnect'
- crawler = get_crawler(SimpleSpider)
- with LogCapture() as l:
- yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
- self._assert_got_response_code(407, l)
-
def _assert_got_response_code(self, code, log):
print(log)
self.assertEqual(str(log).count('Crawled (%d)' % code), 1)
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 27d79437b..24aaaf7ec 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -44,7 +44,7 @@ class BaseRobotParserTest:
def test_allowed_wildcards(self):
robotstxt_robotstxt_body = """User-agent: first
- Disallow: /disallowed/*/end$
+ Disallow: /disallowed/*/end$
User-agent: second
Allow: /*allowed
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 32e65bed5..fda44653a 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,4 +1,3 @@
-import six
import unittest
from unittest import mock
@@ -10,7 +9,7 @@ from . import default_settings
class SettingsGlobalFuncsTest(unittest.TestCase):
def test_get_settings_priority(self):
- for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+ for prio_str, prio_num in SETTINGS_PRIORITIES.items():
self.assertEqual(get_settings_priority(prio_str), prio_num)
self.assertEqual(get_settings_priority(99), 99)
@@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase):
new_dict = {'three': 11, 'four': 21}
attribute.set(new_dict, 10)
self.assertIsInstance(attribute.value, BaseSettings)
- six.assertCountEqual(self, attribute.value, new_dict)
- six.assertCountEqual(self, original_settings, original_dict)
+ self.assertCountEqual(attribute.value, new_dict)
+ self.assertCountEqual(original_settings, original_dict)
new_settings = BaseSettings({'five': 12}, 0)
attribute.set(new_settings, 0) # Insufficient priority
- six.assertCountEqual(self, attribute.value, new_dict)
+ self.assertCountEqual(attribute.value, new_dict)
attribute.set(new_settings, 10)
- six.assertCountEqual(self, attribute.value, new_settings)
+ self.assertCountEqual(attribute.value, new_settings)
def test_repr(self):
self.assertEqual(repr(self.attribute),
@@ -148,10 +147,10 @@ class BaseSettingsTest(unittest.TestCase):
self.settings.setmodule(
'tests.test_settings.default_settings', 10)
- self.assertCountEqual(six.iterkeys(self.settings.attributes),
- six.iterkeys(ctrl_attributes))
+ self.assertCountEqual(self.settings.attributes.keys(),
+ ctrl_attributes.keys())
- for key in six.iterkeys(ctrl_attributes):
+ for key in ctrl_attributes.keys():
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
@@ -227,7 +226,7 @@ class BaseSettingsTest(unittest.TestCase):
}
settings = self.settings
settings.attributes = {key: SettingsAttribute(value, 0) for key, value
- in six.iteritems(test_configuration)}
+ in test_configuration.items()}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
@@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase):
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
s['TEST'].set(2, 200, 'cmdline')
- six.assertCountEqual(self, s.getwithbase('TEST'),
- {1: 1, 2: 200, 3: 30})
- six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+ self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
+ self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertEqual(s.getwithbase('NONEXISTENT'), {})
def test_maxpriority(self):
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index b97d9b675..7511aa568 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,13 +1,11 @@
from unittest import TestCase
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+import warnings
from scrapy.http import Response, Request
from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
-from scrapy.spidermiddlewares.offsite import URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
from scrapy.utils.test import get_crawler
-import warnings
class TestOffsiteMiddleware(TestCase):
@@ -75,7 +73,7 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
-
+
def test_get_host_regex(self):
self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org']
with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 5b7b5e7aa..739cf1c2d 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -156,7 +156,7 @@ class GeneratorFailMiddleware:
r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__))
yield r
raise LookupError()
-
+
def process_spider_exception(self, response, exception, spider):
method = '{}.process_spider_exception'.format(self.__class__.__name__)
spider.logger.info('%s: %s caught', method, exception.__class__.__name__)
@@ -264,7 +264,7 @@ class TestSpiderMiddleware(TestCase):
@classmethod
def tearDownClass(cls):
cls.mockserver.__exit__(None, None, None)
-
+
@defer.inlineCallbacks
def crawl_log(self, spider):
crawler = get_crawler(spider)
@@ -308,7 +308,7 @@ class TestSpiderMiddleware(TestCase):
self.assertIn("{'from': 'errback'}", str(log1))
self.assertNotIn("{'from': 'callback'}", str(log1))
self.assertIn("'item_scraped_count': 1", str(log1))
-
+
@defer.inlineCallbacks
def test_generator_callback(self):
"""
@@ -319,7 +319,7 @@ class TestSpiderMiddleware(TestCase):
log2 = yield self.crawl_log(GeneratorCallbackSpider)
self.assertIn("Middleware: ImportError exception caught", str(log2))
self.assertIn("'item_scraped_count': 2", str(log2))
-
+
@defer.inlineCallbacks
def test_not_a_generator_callback(self):
"""
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 2be6a1cd5..7cc17600c 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
import warnings
@@ -548,8 +548,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
(301, 'http://scrapytest.org/3'),
(301, 'http://scrapytest.org/4'),
),
- b'http://scrapytest.org/1', # expected initial referer
- b'http://scrapytest.org/1', # expected referer for the redirection request
+ b'http://scrapytest.org/1', # expected initial referer
+ b'http://scrapytest.org/1', # expected referer for the redirection request
),
( 'https://scrapytest.org/1',
'https://scrapytest.org/2',
@@ -609,8 +609,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/3'),
(301, 'http://scrapytest.org/4'),
),
- None, # expected initial "Referer"
- None, # expected "Referer" for the redirection request
+ None, # expected initial "Referer"
+ None, # expected "Referer" for the redirection request
),
( 'https://scrapytest.org/1',
'https://scrapytest.org/2',
@@ -648,8 +648,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
- b'http://scrapytest.org/101', # expected initial "Referer"
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b'http://scrapytest.org/101', # expected initial "Referer"
+ b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
@@ -757,8 +757,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
- b'http://scrapytest.org/101', # expected initial referer
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b'http://scrapytest.org/101', # expected initial referer
+ b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
@@ -827,8 +827,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
(301, 'http://scrapytest.org/103'),
(301, 'http://scrapytest.org/104'),
),
- b'http://scrapytest.org/101', # expected initial referer
- b'http://scrapytest.org/101', # expected referer for the redirection request
+ b'http://scrapytest.org/101', # expected initial referer
+ b'http://scrapytest.org/101', # expected referer for the redirection request
),
( 'https://scrapytest.org/201',
'https://scrapytest.org/202',
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 91bbe43bc..fdc5df166 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,12 +1,12 @@
from unittest import TestCase
-import six
+
import scrapy
class ToplevelTestCase(TestCase):
def test_version(self):
- self.assertIs(type(scrapy.__version__), six.text_type)
+ self.assertIs(type(scrapy.__version__), str)
def test_version_info(self):
self.assertIs(type(scrapy.version_info), tuple)
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 22e39821c..bea0cf3e5 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
import unittest
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index c5655df7e..50e1bfd5f 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,7 +1,6 @@
import unittest
import warnings
-from six import assertRaisesRegex
from w3lib.http import basic_auth_header
from scrapy import Request
@@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
def test_too_few_arguments_error(self):
- assertRaisesRegex(
- self,
+ self.assertRaisesRegex(
ValueError,
r"too few arguments|the following arguments are required:\s*url",
lambda: curl_to_request_kwargs("curl"),
@@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
# case 2: ignore_unknown_options=False (raise exception):
- assertRaisesRegex(
- self,
+ self.assertRaisesRegex(
ValueError,
"Unrecognized options:.*--bar.*--baz",
lambda: curl_to_request_kwargs(
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 53228fc6e..38a25778e 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase):
self.assertIn(20, d)
self.assertNotIn(15, d)
- def test_six_range(self):
- import six.moves
- seq = six.moves.range(10**3, 10**6)
- d = SequenceExclude(seq)
- self.assertIn(10**2, d)
- self.assertIn(10**7, d)
- self.assertNotIn(10**4, d)
-
def test_range_step(self):
seq = range(10, 20, 3)
d = SequenceExclude(seq)
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index d642ed3ed..dfbe71ae2 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,8 +5,6 @@ from twisted.python.failure import Failure
from scrapy.utils.defer import mustbe_deferred, process_chain, \
process_chain_both, process_parallel, iter_errback
-from six.moves import xrange
-
class MustbeDeferredTest(unittest.TestCase):
def test_success_function(self):
@@ -16,8 +14,8 @@ class MustbeDeferredTest(unittest.TestCase):
return steps
dfd = mustbe_deferred(_append, 1)
- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
- steps.append(2) # add another value, that should be catched by assertEqual
+ dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
+ steps.append(2) # add another value, that should be catched by assertEqual
return dfd
def test_unfired_deferred(self):
@@ -29,8 +27,8 @@ class MustbeDeferredTest(unittest.TestCase):
return dfd
dfd = mustbe_deferred(_append, 1)
- dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
- steps.append(2) # add another value, that should be catched by assertEqual
+ dfd.addCallback(self.assertEqual, [1, 2]) # it is [1] with maybeDeferred
+ steps.append(2) # add another value, that should be catched by assertEqual
return dfd
@@ -92,7 +90,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
- for x in xrange(10):
+ for x in range(10):
yield x
errors = []
@@ -102,7 +100,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_bad(self):
def iterbad():
- for x in xrange(10):
+ for x in range(10):
if x == 5:
a = 1/0
yield x
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index ce04e7f29..159ef8f25 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import inspect
import unittest
from unittest import mock
diff --git a/tests/test_utils_http.py b/tests/test_utils_http.py
index f9af4bf87..2fac3da1f 100644
--- a/tests/test_utils_http.py
+++ b/tests/test_utils_http.py
@@ -13,7 +13,7 @@ class ChunkedTest(unittest.TestCase):
chunked_body += "8\r\n" + "sequence\r\n"
chunked_body += "0\r\n\r\n"
body = decode_chunked_transfer(chunked_body)
- self.assertEqual(body, \
- "This is the data in the first chunk\r\n" +
- "and this is the second one\r\n" +
- "consequence")
+ self.assertEqual(body,
+ "This is the data in the first chunk\r\n" +
+ "and this is the second one\r\n" +
+ "consequence")
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 2c3965bbc..cf8ad1f23 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,5 +1,5 @@
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f16ef8110..9776dfb2a 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
import os
-import six
+
from twisted.trial import unittest
from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
from scrapy.http import XmlResponse, TextResponse, Response
from tests import get_testdata
+
FOOBAR_NL = u"foo\nbar"
@@ -256,8 +257,8 @@ class UtilsCsvTestCase(unittest.TestCase):
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys())))
- self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values())))
+ self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
+ self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
def test_csviter_delimiter(self):
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 742e04803..2c23f3616 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import print_function
import sys
import logging
import unittest
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 2d27d4b81..b79e0ac1c 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,10 +1,9 @@
-import gc
import functools
+import gc
import operator
+import platform
import unittest
from itertools import count
-import platform
-import six
from warnings import catch_warnings
from scrapy.utils.python import (
@@ -12,6 +11,7 @@ from scrapy.utils.python import (
WeakKeyCache, get_func_args, to_bytes, to_unicode,
without_none_values, MutableChain)
+
__doctests__ = ['scrapy.utils.python']
@@ -205,12 +205,12 @@ class UtilsPythonTestCase(unittest.TestCase):
if platform.python_implementation() == 'CPython':
# TODO: how do we fix this to return the actual argument names?
- self.assertEqual(get_func_args(six.text_type.split), [])
+ self.assertEqual(get_func_args(str.split), [])
self.assertEqual(get_func_args(" ".join), [])
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
else:
self.assertEqual(
- get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit'])
+ get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
self.assertEqual(
get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 3da95b95a..3e664fc74 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import unittest
from scrapy.http import Request
from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index bea4dade3..6ebf290c0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,12 +1,13 @@
import os
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
get_meta_refresh, get_base_url, response_status_message)
+
__doctests__ = ['scrapy.utils.response']
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index edeeacc80..ee7d17062 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -1,20 +1,16 @@
import unittest
+
+from scrapy import Spider
from scrapy.http import Request
from scrapy.item import BaseItem
from scrapy.utils.spider import iterate_spider_output, iter_spider_classes
-from scrapy.spiders import CrawlSpider
-
-class MyBaseSpider(CrawlSpider):
- pass # abstract spider
-
-
-class MySpider1(MyBaseSpider):
+class MySpider1(Spider):
name = 'myspider1'
-class MySpider2(MyBaseSpider):
+class MySpider2(Spider):
name = 'myspider2'
@@ -35,5 +31,6 @@ class UtilsSpidersTestCase(unittest.TestCase):
it = iter_spider_classes(tests.test_utils_spider)
self.assertEqual(set(it), {MySpider1, MySpider2})
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 480a717e7..16e02f919 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,6 +1,7 @@
-import six
import unittest
+from io import StringIO
from unittest import mock
+
from scrapy.utils import trackref
@@ -38,12 +39,12 @@ Live References
Bar 1 oldest: 0s ago
''')
- @mock.patch('sys.stdout', new_callable=six.StringIO)
+ @mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_empty(self, stdout):
trackref.print_live_refs()
self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
- @mock.patch('sys.stdout', new_callable=six.StringIO)
+ @mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_with_objects(self, stdout):
o1 = Foo() # NOQA
trackref.print_live_refs()
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index c7bcaf88b..21e9a056a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -5,6 +5,7 @@ from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
add_http_if_no_scheme, guess_scheme, strip_url)
+
__doctests__ = ['scrapy.utils.url']
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 7b015ff8d..746367b41 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,7 +3,6 @@ from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
import os
-import six
import shutil
import OpenSSL.SSL
@@ -298,7 +297,7 @@ class WebClientTestCase(unittest.TestCase):
def cleanup(passthrough):
# Clean up the server which is hanging around not doing
# anything.
- connected = list(six.iterkeys(self.wrapper.protocols))
+ connected = list(self.wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected:
diff --git a/tox.ini b/tox.ini
index fd75d18e2..1ec8f52e4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -4,12 +4,12 @@
# and then run "tox" from this directory.
[tox]
-envlist = py35
+envlist = security,flake8,py3
+minversion = 1.7.0
[testenv]
deps =
-ctests/constraints.txt
- -rrequirements-py3.txt
-rtests/requirements-py3.txt
# Extras
botocore>=1.3.23
@@ -23,11 +23,28 @@ passenv =
commands =
py.test --cov=scrapy --cov-report= {posargs:--durations=10 docs scrapy tests}
-[testenv:py35]
-basepython = python3.5
+[testenv:security]
+basepython = python3
+deps =
+ bandit
+commands =
+ bandit -r -c .bandit.yml {posargs:scrapy}
-[testenv:py35-pinned]
-basepython = python3.5
+[testenv:flake8]
+basepython = python3
+deps =
+ {[testenv]deps}
+ pytest-flake8
+commands =
+ py.test --flake8 {posargs:docs scrapy tests}
+
+[testenv:pypy3]
+basepython = pypy3
+commands =
+ py.test {posargs:--durations=10 docs scrapy tests}
+
+[testenv:pinned]
+basepython = python3
deps =
-ctests/constraints.txt
cryptography==2.0
@@ -48,34 +65,11 @@ deps =
botocore==1.3.23
Pillow==3.4.2
-[testenv:py36]
-basepython = python3.6
-
-[testenv:py37]
-basepython = python3.7
-
-[testenv:py38]
-basepython = python3.8
-
-[testenv:pypy3]
-basepython = pypy3
-commands =
- py.test {posargs:--durations=10 docs scrapy tests}
-
-[testenv:security]
-basepython = python3.8
-deps =
- bandit
-commands =
- bandit -r -c .bandit.yml {posargs:scrapy}
-
-[testenv:flake8]
-basepython = python3.8
+[testenv:extra-deps]
deps =
{[testenv]deps}
- pytest-flake8
-commands =
- py.test --flake8 {posargs:docs scrapy tests}
+ reppy
+ robotexclusionrulesparser
[docs]
changedir = docs
@@ -83,26 +77,22 @@ deps =
-rdocs/requirements.txt
[testenv:docs]
+basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
commands =
sphinx-build -W -b html . {envtmpdir}/html
[testenv:docs-coverage]
+basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
commands =
sphinx-build -b coverage . {envtmpdir}/coverage
[testenv:docs-links]
+basepython = python3
changedir = {[docs]changedir}
deps = {[docs]deps}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
-
-[testenv:py38-extra-deps]
-basepython = python3.8
-deps =
- {[testenv]deps}
- reppy
- robotexclusionrulesparser