diff --git a/.bandit.yml b/.bandit.yml
index 00554587a..243379b0b 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,13 +1,15 @@
skips:
- B101
- B105
+- B301
- B303
- B306
- B307
- B311
- B320
- B321
-- B402
+- B402 # https://github.com/scrapy/scrapy/issues/4180
+- B403
- B404
- B406
- B410
diff --git a/conftest.py b/conftest.py
index 24e31f130..64136b48d 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,12 +1,19 @@
+from pathlib import Path
+
import pytest
+def _py_files(folder):
+ return (str(p) for p in Path(folder).rglob('*.py'))
+
+
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
+ # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+ *_py_files("tests/CrawlerProcess")
]
-
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 8b2fef065..01986b594 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
def parse(self, response):
for quote in response.css('div.quote'):
yield {
- 'text': quote.css('span.text::text').get(),
'author': quote.xpath('span/small/text()').get(),
+ 'text': quote.css('span.text::text').get(),
}
next_page = response.css('li.next a::attr("href")').get()
diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 3bef20bf3..da7a0022b 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -1,5 +1,4 @@
#!/usr/bin/env python
-from __future__ import print_function
from time import time
from collections import deque
from twisted.web.server import Site, NOT_DONE_YET
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 2c98e6f6d..28c649e28 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -2,7 +2,6 @@ parsel>=1.5.0
PyDispatcher>=2.0.5
Twisted>=17.9.0
w3lib>=1.17.0
-protego>=0.1.15
pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
@@ -14,5 +13,4 @@ cryptography>=2.0 # Earlier versions would fail to install
cssselect>=0.9.1
lxml>=3.5.0
service_identity>=16.0.0
-six>=1.10.0
zope.interface>=4.1.3
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 230e5cee3..fb8357f3c 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
del warnings
# Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
del _monkeypatches
from twisted import version as _txv
diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py
index 1f8067b35..f74f89bda 100644
--- a/scrapy/_monkeypatches.py
+++ b/scrapy/_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves import copyreg
+import copyreg
# Undo what Twisted's perspective broker adds to pickle register
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index ce030cf75..3c2efe58f 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import sys
import os
import optparse
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 90c8d56a2..7bbe362e7 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,8 +1,7 @@
import sys
import time
import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
import scrapy
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 724b4a1c4..8a22ebabe 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import sys
from w3lib.url import is_url
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index d5498bb5c..adb01fa70 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import os
import shutil
import string
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 422183ac1..54d7bb228 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ef8acd29c..ff6f1d8cd 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import json
import logging
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index ffe3aa2eb..603bafb9f 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import json
from scrapy.commands import ScrapyCommand
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index 3b9f6eabb..e65131ae8 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import re
import os
import string
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 8651948f7..1516c5997 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import scrapy
from scrapy.commands import ScrapyCommand
from scrapy.utils.versions import scrapy_components_versions
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 24f6c2e77..e0d425874 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -4,7 +4,7 @@ from scrapy.item import BaseItem
from scrapy.http import Request
from scrapy.exceptions import ContractFail
-from . import Contract
+from scrapy.contracts import Contract
# contracts
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 213268741..157dc3418 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -3,15 +3,14 @@ from time import time
from datetime import datetime
from collections import deque
-import six
from twisted.internet import reactor, defer, task
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.httpobj import urlparse_cached
from scrapy.resolver import dnscache
from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
class Slot(object):
@@ -188,7 +187,7 @@ class Downloader(object):
def close(self):
self._slot_gc_loop.stop()
- for slot in six.itervalues(self.slots):
+ for slot in self.slots.values():
slot.close()
def _slot_gc(self, age=60):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 0b55d32fa..39a0b1f51 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,8 +1,9 @@
"""Download handlers for different schemes"""
import logging
+
from twisted.internet import defer
-import six
+
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
self._notconfigured = {} # remembers failed handlers
handlers = without_none_values(
crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
- for scheme, clspath in six.iteritems(handlers):
+ for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 39ed67a1a..aef231e82 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
import re
from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from twisted.internet import reactor
from twisted.protocols.ftp import FTPClient, CommandFailed
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index 6111e132a..52535bd8b 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,2 +1,4 @@
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+ HTTP11DownloadHandler as HTTPDownloadHandler,
+)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 7d917cb74..63dedc19b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -2,10 +2,10 @@
import re
import logging
+import warnings
from io import BytesIO
from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
from zope.interface import implementer
from twisted.internet import defer, reactor, protocol
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 808d1bf21..e2a07bdef 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,9 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
def _get_boto_connection():
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 7a6a4dfac..38608a429 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -3,8 +3,6 @@ Downloader Middleware manager
See documentation in docs/topics/downloader-middleware.rst
"""
-import six
-
from twisted.internet import defer
from scrapy.exceptions import _InvalidOutput
@@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
+ (method.__self__.__class__.__name__, response.__class__.__name__))
if response:
defer.returnValue(response)
defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, response=response, spider=spider)
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response)))
+ (method.__self__.__class__.__name__, type(response)))
if isinstance(response, Request):
defer.returnValue(response)
defer.returnValue(response)
@@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
response = yield method(request=request, exception=exception, spider=spider)
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
- (six.get_method_self(method).__class__.__name__, type(response)))
+ (method.__self__.__class__.__name__, type(response)))
if response:
defer.returnValue(response)
defer.returnValue(_failure)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3fe13414a..798346f19 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,5 +1,5 @@
from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 00cee3ada..e2ade8256 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
"""
from itertools import chain, islice
-import six
from twisted.python.failure import Failure
from scrapy.exceptions import _InvalidOutput
from scrapy.middleware import MiddlewareManager
@@ -37,8 +36,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f: '%s.%s' % (
- six.get_method_self(f).__class__.__name__,
- six.get_method_function(f).__name__)
+ f.__self__.__class__.__name__,
+ f.__func__.__name__)
def process_spider_input(response):
for method in self.methods['process_spider_input']:
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 19b61dc7e..6c7eb737b 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -1,7 +1,6 @@
-import pprint
-import six
-import signal
import logging
+import pprint
+import signal
import warnings
from twisted.internet import reactor, defer
@@ -22,6 +21,7 @@ from scrapy.utils.log import (
get_scrapy_root_handler, install_scrapy_root_handler)
from scrapy import signals
+
logger = logging.getLogger(__name__)
@@ -206,7 +206,7 @@ class CrawlerRunner(object):
return self._create_crawler(crawler_or_spidercls)
def _create_crawler(self, spidercls):
- if isinstance(spidercls, six.string_types):
+ if isinstance(spidercls, str):
spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings)
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index ba50793bb..7a140fcad 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -1,9 +1,7 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import re
import logging
-import six
from w3lib import html
from scrapy.exceptions import NotConfigured
@@ -67,7 +65,7 @@ class AjaxCrawlMiddleware(object):
# XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r''))
+_ajax_crawlable_re = re.compile(r'')
def _has_ajaxcrawlable_meta(text):
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index aeb7578b8..d8dabdf13 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -1,13 +1,12 @@
import logging
from collections import defaultdict
-import six
-
from scrapy.exceptions import NotConfigured
from scrapy.http import Response
from scrapy.http.cookies import CookieJar
from scrapy.utils.python import to_unicode
+
logger = logging.getLogger(__name__)
@@ -82,8 +81,10 @@ class CookiesMiddleware(object):
def _get_request_cookies(self, jar, request):
if isinstance(request.cookies, dict):
- cookie_list = [{'name': k, 'value': v} for k, v in \
- six.iteritems(request.cookies)]
+ cookie_list = [
+ {'name': k, 'value': v}
+ for k, v in request.cookies.items()
+ ]
else:
cookie_list = request.cookies
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index e2d73f347..fcea38ef5 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
import bz2
import gzip
-from io import BytesIO
-import zipfile
-import tarfile
import logging
+import tarfile
+import zipfile
+from io import BytesIO
from tempfile import mktemp
-import six
-
from scrapy.responsetypes import responsetypes
+
logger = logging.getLogger(__name__)
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
if not response.body:
return response
- for fmt, func in six.iteritems(self._formats):
+ for fmt, func in self._formats.items():
new_response = func(response)
if new_response:
logger.debug('Decompressed response with format: %(responsefmt)s',
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 5e4542b6c..814ce78fe 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -1,8 +1,6 @@
import base64
-from urllib.request import _parse_proxy
-
-from six.moves.urllib.parse import unquote, urlunparse
-from six.moves.urllib.request import getproxies, proxy_bypass
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index b73f864dd..77cb5aa94 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,5 +1,5 @@
import logging
-from six.moves.urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin, urlparse
from w3lib.url import safe_url_string
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
from scrapy.utils.response import get_meta_refresh
from scrapy.exceptions import IgnoreRequest, NotConfigured
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 4d95eb847..ea6a4cfc3 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import os
import logging
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 3defafd60..fcb55da67 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -6,15 +6,14 @@ import csv
import io
import pprint
import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+import pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
from scrapy.utils.python import to_bytes, to_unicode, is_listlike
from scrapy.item import BaseItem
from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@@ -60,9 +59,9 @@ class BaseItemExporter(object):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
if include_empty and not isinstance(item, dict):
- field_iter = six.iterkeys(item.fields)
+ field_iter = item.fields.keys()
else:
- field_iter = six.iterkeys(item)
+ field_iter = item.keys()
else:
if include_empty:
field_iter = self.fields_to_export
@@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter):
for value in serialized_value:
self._export_xml_field('value', value, depth=depth+1)
self._beautify_indent(depth=depth)
- elif isinstance(serialized_value, six.text_type):
+ elif isinstance(serialized_value, str):
self.xg.characters(serialized_value)
else:
self.xg.characters(str(serialized_value))
@@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
if is_listlike(value):
return [self._serialize_value(v) for v in value]
encode_func = to_bytes if self.binary else to_unicode
- if isinstance(value, (six.text_type, bytes)):
+ if isinstance(value, (str, bytes)):
return encode_func(value, encoding=self.encoding)
return value
def _serialize_dict(self, value):
- for key, val in six.iteritems(value):
+ for key, val in value.items():
key = to_bytes(key) if self.binary else key
yield key, self._serialize_value(val)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e2492d506..11a63ced2 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -10,7 +10,7 @@ import logging
import posixpath
from tempfile import NamedTemporaryFile
from datetime import datetime
-from six.moves.urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote
from ftplib import FTP
from zope.interface import Interface, implementer
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 11403957c..91850683f 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,14 +1,12 @@
-from __future__ import print_function
-
import gzip
import logging
import os
+import pickle
from email.utils import mktime_tz, parsedate_tz
from importlib import import_module
from time import time
from weakref import WeakKeyDictionary
-from six.moves import cPickle as pickle
from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
from scrapy.http import Headers, Response
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 263d8ce4c..892aa8a86 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
"""
import gc
-import six
from scrapy import signals
from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
def spider_closed(self, spider, reason):
gc.collect()
self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if not wdict:
continue
self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 8ba770ec0..2c8e46914 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,5 +1,5 @@
import os
-from six.moves import cPickle as pickle
+import pickle
from scrapy import signals
from scrapy.exceptions import NotConfigured
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 60a14c6f8..0903fd4f8 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,7 +1,6 @@
import time
-from six.moves.http_cookiejar import (
- CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index f3b46b994..dcaaeddfa 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,4 +1,3 @@
-import six
from w3lib.http import headers_dict_to_raw
from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.python import to_unicode
@@ -19,7 +18,7 @@ class Headers(CaselessDict):
"""Normalize values to bytes"""
if value is None:
value = []
- elif isinstance(value, (six.text_type, bytes)):
+ elif isinstance(value, (str, bytes)):
value = [value]
elif not hasattr(value, '__iter__'):
value = [value]
@@ -29,10 +28,10 @@ class Headers(CaselessDict):
def _tobytes(self, x):
if isinstance(x, bytes):
return x
- elif isinstance(x, six.text_type):
+ elif isinstance(x, str):
return x.encode(self.encoding)
elif isinstance(x, int):
- return six.text_type(x).encode(self.encoding)
+ return str(x).encode(self.encoding)
else:
raise TypeError('Unsupported value type: {}'.format(type(x)))
@@ -68,9 +67,6 @@ class Headers(CaselessDict):
self[key] = lst
def items(self):
- return list(self.iteritems())
-
- def iteritems(self):
return ((k, self.getlist(k)) for k in self.keys())
def values(self):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 76a428199..b5c8e1a9a 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,7 +4,6 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-import six
from w3lib.url import safe_url_string
from scrapy.http.headers import Headers
@@ -60,7 +59,7 @@ class Request(object_ref):
return self._url
def _set_url(self, url):
- if not isinstance(url, six.string_types):
+ if not isinstance(url, str):
raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
s = safe_url_string(url, self.encoding)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index b6feede07..af02c8484 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
import lxml.html
from parsel.selector import create_root_node
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
# We didn't find it, so now we build an XPath expression out of the other
# arguments, because they can be used as such
xpath = u'.//*' + \
- u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+ u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
el = form.xpath(xpath)
if len(el) == 1:
return (el[0].get('name'), el[0].get('value') or '')
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index bd09f7534..811d3ad6b 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
from scrapy.http.request import Request
from scrapy.utils.python import get_func_args
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index a81404afb..64e9c6c20 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -4,7 +4,7 @@ responses in Scrapy.
See documentation in docs/topics/request-response.rst
"""
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from scrapy.http.request import Request
from scrapy.http.headers import Headers
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 37f450e54..1079fd6e8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
See documentation in docs/topics/request-response.rst
"""
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
import parsel
from w3lib.encoding import html_to_unicode, resolve_encoding, \
@@ -31,14 +30,14 @@ class TextResponse(Response):
super(TextResponse, self).__init__(*args, **kwargs)
def _set_url(self, url):
- if isinstance(url, six.text_type):
+ if isinstance(url, str):
self._url = to_unicode(url, self.encoding)
else:
super(TextResponse, self)._set_url(url)
def _set_body(self, body):
self._body = b'' # used by encoding detection
- if isinstance(body, six.text_type):
+ if isinstance(body, str):
if self._encoding is None:
raise TypeError('Cannot convert unicode body - %s has no encoding' %
type(self).__name__)
@@ -158,7 +157,7 @@ class TextResponse(Response):
def _url_from_selector(sel):
# type: (parsel.Selector) -> str
- if isinstance(sel.root, six.string_types):
+ if isinstance(sel.root, str):
# e.g. ::attr(href) result
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, 'tag'):
diff --git a/scrapy/item.py b/scrapy/item.py
index 32f9b2ebb..1d39b48b2 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -10,8 +10,6 @@ from copy import deepcopy
from pprint import pformat
from warnings import warn
-import six
-
from scrapy.utils.deprecate import ScrapyDeprecationWarning
from scrapy.utils.trackref import object_ref
@@ -78,7 +76,7 @@ class DictItem(MutableMapping, BaseItem):
def __init__(self, *args, **kwargs):
self._values = {}
if args or kwargs: # avoid creating dict for most common case
- for k, v in six.iteritems(dict(*args, **kwargs)):
+ for k, v in dict(*args, **kwargs).items():
self[k] = v
def __getitem__(self, key):
@@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
return deepcopy(self)
-@six.add_metaclass(ItemMeta)
-class Item(DictItem):
+class Item(DictItem, metaclass=ItemMeta):
pass
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 8c3693f04..4a3e74fbe 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
For more info see docs/topics/link-extractors.rst
"""
import re
+from urllib.parse import urlparse
-from six.moves.urllib.parse import urlparse
from parsel.csstranslator import HTMLTranslator
from w3lib.url import canonicalize_url
@@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):
# Top-level imports
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor # noqa: F401
diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py
index 27978a8a1..0425d4340 100644
--- a/scrapy/linkextractors/htmlparser.py
+++ b/scrapy/linkextractors/htmlparser.py
@@ -2,9 +2,8 @@
HTMLParser-based link extractor
"""
import warnings
-import six
-from six.moves.html_parser import HTMLParser
-from six.moves.urllib.parse import urljoin
+from html.parser import HTMLParser
+from urllib.parse import urljoin
from w3lib.url import safe_url_string
from w3lib.html import strip_html5_whitespace
@@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser):
ret = []
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 890c019c8..cb55e805a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -1,8 +1,7 @@
"""
Link extractor based on lxml.html
"""
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
@@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()")
def _nons(tag):
- if isinstance(tag, six.string_types):
+ if isinstance(tag, str):
if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
return tag.split('}')[-1]
return tag
diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py
index e689b4727..3f2557248 100644
--- a/scrapy/linkextractors/regex.py
+++ b/scrapy/linkextractors/regex.py
@@ -1,10 +1,11 @@
import re
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
from scrapy.link import Link
-from .sgml import SgmlLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor
+
linkre = re.compile(
"|\s.*?>)(.*?)<[/ ]?a>",
diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py
index 8940a4d77..2ba6bca45 100644
--- a/scrapy/linkextractors/sgml.py
+++ b/scrapy/linkextractors/sgml.py
@@ -1,9 +1,8 @@
"""
SGMLParser-based Link extractors
"""
-import six
-from six.moves.urllib.parse import urljoin
import warnings
+from urllib.parse import urljoin
from sgmllib import SGMLParser
from w3lib.url import safe_url_string, canonicalize_url
@@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
- if isinstance(link.url, six.text_type):
+ if isinstance(link.url, str):
link.url = link.url.encode(response_encoding)
try:
link.url = urljoin(base_url, link.url)
diff --git a/scrapy/mail.py b/scrapy/mail.py
index d24de2212..891bb5e09 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -3,21 +3,21 @@ Mail sending helpers
See documentation in docs/topics/email.rst
"""
-from io import BytesIO
import logging
-
-from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-from email.mime.nonmultipart import MIMENonMultipart
from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from io import BytesIO
from twisted.internet import defer, reactor, ssl
from scrapy.utils.misc import arg_to_iter
from scrapy.utils.python import to_bytes
+
logger = logging.getLogger(__name__)
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8d74c5011..6d55c8980 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
-from io import BytesIO
+import logging
import mimetypes
import os
-import os.path
import time
-import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
from collections import defaultdict
-import six
+from email.utils import parsedate_tz, mktime_tz
+from io import BytesIO
+from urllib.parse import urlparse
from twisted.internet import defer, threads
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
from scrapy.utils.boto import is_botocore
from scrapy.utils.datatypes import CaselessDict
+
logger = logging.getLogger(__name__)
@@ -153,14 +152,14 @@ class S3FilesStore(object):
Bucket=self.bucket,
Key=key_name,
Body=buf,
- Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+ Metadata={k: str(v) for k, v in (meta or {}).items()},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
- for metakey, metavalue in six.iteritems(meta):
+ for metakey, metavalue in meta.items():
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
})
extra = {}
- for key, value in six.iteritems(headers):
+ for key, value in headers.items():
try:
kwarg = mapping[key]
except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
def persist_file(self, path, buf, info, meta=None, headers=None):
blob = self.bucket.blob(self.prefix + path)
blob.cache_control = self.CACHE_CONTROL
- blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+ blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
return threads.deferToThread(
blob.upload_from_string,
data=buf.getvalue(),
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e77cef4ff..e9c6b759c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
import functools
import hashlib
from io import BytesIO
-import six
from PIL import Image
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
image, buf = self.convert_image(orig_image)
yield path, image, buf
- for thumb_id, size in six.iteritems(self.thumbs):
+ for thumb_id, size in self.thumbs.items():
thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
thumb_image, thumb_buf = self.convert_image(image, size)
yield thumb_path, thumb_image, thumb_buf
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 95dca9a3f..c174addf9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
import functools
import logging
from collections import defaultdict
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index de62276c8..91d309147 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -2,11 +2,9 @@
This module implements a class which returns the appropriate Response class
based on different criteria.
"""
-from __future__ import absolute_import
from mimetypes import MimeTypes
from pkgutil import get_data
from io import StringIO
-import six
from scrapy.http import Response
from scrapy.utils.misc import load_object
@@ -37,7 +35,7 @@ class ResponseTypes(object):
self.mimetypes = MimeTypes()
mimedata = get_data('scrapy', 'mime.types').decode('utf8')
self.mimetypes.readfp(StringIO(mimedata))
- for mimetype, cls in six.iteritems(self.CLASSES):
+ for mimetype, cls in self.CLASSES.items():
self.classes[mimetype] = load_object(cls)
def from_mimetype(self, mimetype):
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index f0f9c59dc..0a9af3a62 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -1,7 +1,6 @@
import sys
import logging
from abc import ABCMeta, abstractmethod
-from six import with_metaclass
from scrapy.utils.python import to_unicode
@@ -26,7 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
return robotstxt_body
-class RobotParser(with_metaclass(ABCMeta)):
+class RobotParser(metaclass=ABCMeta):
@classmethod
@abstractmethod
def from_crawler(cls, crawler, robotstxt_body):
@@ -56,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)):
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body, spider):
- from six.moves.urllib_robotparser import RobotFileParser
+ from urllib.robotparser import RobotFileParser
self.spider = spider
robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
self.rp = RobotFileParser()
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index c871e86e0..b6133619c 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,4 +1,3 @@
-import six
import json
import copy
from collections.abc import MutableMapping
@@ -23,7 +22,7 @@ def get_settings_priority(priority):
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
numerical value, or directly returns a given numerical priority.
"""
- if isinstance(priority, six.string_types):
+ if isinstance(priority, str):
return SETTINGS_PRIORITIES[priority]
else:
return priority
@@ -173,7 +172,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or [])
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = value.split(',')
return list(value)
@@ -194,7 +193,7 @@ class BaseSettings(MutableMapping):
:type default: any
"""
value = self.get(name, default or {})
- if isinstance(value, six.string_types):
+ if isinstance(value, str):
value = json.loads(value)
return dict(value)
@@ -284,7 +283,7 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(module, six.string_types):
+ if isinstance(module, str):
module = import_module(module)
for key in dir(module):
if key.isupper():
@@ -313,14 +312,14 @@ class BaseSettings(MutableMapping):
:type priority: string or int
"""
self._assert_mutability()
- if isinstance(values, six.string_types):
+ if isinstance(values, str):
values = json.loads(values)
if values is not None:
if isinstance(values, BaseSettings):
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, values.getpriority(name))
else:
- for name, value in six.iteritems(values):
+ for name, value in values.items():
self.set(name, value, priority)
def delete(self, name, priority='project'):
@@ -377,7 +376,7 @@ class BaseSettings(MutableMapping):
def _to_dict(self):
return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
- for k, v in six.iteritems(self)}
+ for k, v in self.items()}
def copy_to_dict(self):
"""
@@ -445,7 +444,7 @@ class Settings(BaseSettings):
self.setmodule(default_settings, 'default')
# Promote default dictionaries to BaseSettings instances for per-key
# priorities
- for name, val in six.iteritems(self):
+ for name, val in self.items():
if isinstance(val, dict):
self.set(name, BaseSettings(val, 'default'), 'default')
self.update(values, priority)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 80b625633..a649d555f 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -3,8 +3,6 @@
See documentation in docs/topics/shell.rst
"""
-from __future__ import print_function
-
import os
import signal
import warnings
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 9a160f62e..481d97e9a 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
from pydispatch import dispatcher
from scrapy.utils import signal as _signal
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 7478faa78..3beca4060 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
from collections import defaultdict
import traceback
import warnings
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index c76e4d5a2..dce2b3598 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -2,8 +2,8 @@
RefererMiddleware: populates Request referer field, based on the Response which
originated it.
"""
-from six.moves.urllib.parse import urlparse
import warnings
+from urllib.parse import urlparse
from w3lib.url import safe_url_string
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 8d15dfceb..9429f6cb2 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -57,6 +57,11 @@ class Spider(object_ref):
def start_requests(self):
cls = self.__class__
+ if not self.start_urls and hasattr(self, 'start_url'):
+ raise AttributeError(
+ "Crawling could not start: 'start_urls' not found "
+ "or empty (but found 'start_url' attribute instead, "
+ "did you miss an 's'?)")
if method_is_overridden(cls, Spider, 'make_requests_from_url'):
warnings.warn(
"Spider.make_requests_from_url method is deprecated; it "
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 03000ce54..a5eb1a518 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst
import copy
import warnings
-import six
-
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, HtmlResponse
from scrapy.linkextractors import LinkExtractor
@@ -25,7 +23,7 @@ def _identity(request, response):
def _get_method(method, spider):
if callable(method):
return method
- elif isinstance(method, six.string_types):
+ elif isinstance(method, str):
return getattr(spider, method, None)
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 534c45c70..d368c7108 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,6 +1,5 @@
import re
import logging
-import six
from scrapy.spiders import Spider
from scrapy.http import Request, XmlResponse
@@ -22,7 +21,7 @@ class SitemapSpider(Spider):
super(SitemapSpider, self).__init__(*a, **kw)
self._cbs = []
for r, c in self.sitemap_rules:
- if isinstance(c, six.string_types):
+ if isinstance(c, str):
c = getattr(self, c)
self._cbs.append((regex(r), c))
self._follow = [regex(x) for x in self.sitemap_follow]
@@ -86,7 +85,7 @@ class SitemapSpider(Spider):
def regex(x):
- if isinstance(x, six.string_types):
+ if isinstance(x, str):
return re.compile(x)
return x
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 30cc926e5..d5d3be67e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,7 +3,7 @@ Scheduler queues
"""
import marshal
-from six.moves import cPickle as pickle
+import pickle
from queuelib import queue
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 5bbda6e27..cdbe21942 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -1,5 +1,6 @@
import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.internet import reactor
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index b76d5e56e..12321caa5 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,5 @@
"""Boto/botocore helpers"""
-from __future__ import absolute_import
-
from scrapy.exceptions import NotConfigured
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 561bb72fc..7a15e77ff 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -1,11 +1,9 @@
-from configparser import ConfigParser
import os
import sys
import numbers
+from configparser import ConfigParser
from operator import itemgetter
-import six
-
from scrapy.settings import BaseSettings
from scrapy.utils.deprecate import update_classpath
from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
def _map_keys(compdict):
if isinstance(compdict, BaseSettings):
compbs = BaseSettings()
- for k, v in six.iteritems(compdict):
+ for k, v in compdict.items():
prio = compdict.getpriority(k)
if compbs.getpriority(convert(k)) == prio:
raise ValueError('Some paths in {!r} convert to the same '
@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
return compbs
else:
_check_components(compdict)
- return {convert(k): v for k, v in six.iteritems(compdict)}
+ return {convert(k): v for k, v in compdict.items()}
def _validate_values(compdict):
"""Fail if a value in the components dict is not a real number or None."""
- for name, value in six.iteritems(compdict):
+ for name, value in compdict.items():
if value is not None and not isinstance(value, numbers.Real):
raise ValueError('Invalid value {} for component {}, please provide ' \
'a real number or None instead'.format(value, name))
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
_validate_values(compdict)
compdict = without_none_values(_map_keys(compdict))
- return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+ return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
def arglist_to_dict(arglist):
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 7fb25a71d..16639356e 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -1,10 +1,9 @@
import argparse
import warnings
from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
-from six.moves.http_cookies import SimpleCookie
-from six.moves.urllib.parse import urlparse
-from six import iteritems
from w3lib.http import basic_auth_header
@@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
name = name.strip()
val = val.strip()
if name.title() == 'Cookie':
- for name, morsel in iteritems(SimpleCookie(val)):
+ for name, morsel in SimpleCookie(val).items():
cookies[name] = morsel.value
else:
headers.append((name, val))
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 39d389fa6..ffd1537c3 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -5,12 +5,10 @@ Python Standard Library.
This module must not depend on any module outside the Standard Library.
"""
-import copy
import collections
-from collections.abc import Mapping
+import copy
import warnings
-
-import six
+from collections.abc import Mapping
from scrapy.exceptions import ScrapyDeprecationWarning
@@ -151,7 +149,7 @@ class MultiValueDict(dict):
self.setlistdefault(key, []).append(value)
except TypeError:
raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
- for key, value in six.iteritems(kwargs):
+ for key, value in kwargs.items():
self.setlistdefault(key, []).append(value)
@@ -238,65 +236,6 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
-class MergeDict(object):
- """
- A simple class for creating new "virtual" dictionaries that actually look
- up values in more than one dictionary, passed in the ``__init__`` method.
-
- If a key appears in more than one of the given dictionaries, only the
- first occurrence will be used.
- """
- def __init__(self, *dicts):
- warnings.warn(
- "scrapy.utils.datatypes.MergeDict is deprecated in favor "
- "of collections.ChainMap (introduced in Python 3.3)",
- category=ScrapyDeprecationWarning,
- stacklevel=2,
- )
- self.dicts = dicts
-
- def __getitem__(self, key):
- for dict_ in self.dicts:
- try:
- return dict_[key]
- except KeyError:
- pass
- raise KeyError
-
- def __copy__(self):
- return self.__class__(*self.dicts)
-
- def get(self, key, default=None):
- try:
- return self[key]
- except KeyError:
- return default
-
- def getlist(self, key):
- for dict_ in self.dicts:
- if key in dict_.keys():
- return dict_.getlist(key)
- return []
-
- def items(self):
- item_list = []
- for dict_ in self.dicts:
- item_list.extend(dict_.items())
- return item_list
-
- def has_key(self, key):
- for dict_ in self.dicts:
- if key in dict_:
- return True
- return False
-
- __contains__ = has_key
-
- def copy(self):
- """Returns a copy of this object."""
- return self.__copy__()
-
-
class LocalCache(collections.OrderedDict):
"""Dictionary with a finite number of keys.
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index 91ebdae11..9735220ef 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -2,7 +2,6 @@
pprint and pformat wrappers with colorization support
"""
-from __future__ import print_function
import sys
from pprint import pformat as pformat_
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index b2be0a901..c8d4391b1 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,8 +1,7 @@
"""Helper functions for scrapy.http objects (Request, Response)"""
import weakref
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
_urlparse_cache = weakref.WeakKeyDictionary()
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9693ba768..3c0cb68c3 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -1,13 +1,13 @@
-import re
import csv
-from io import StringIO
import logging
-import six
+import re
+from io import StringIO
from scrapy.http import TextResponse, Response
from scrapy.selector import Selector
from scrapy.utils.python import re_rsearch, to_unicode
+
logger = logging.getLogger(__name__)
@@ -60,7 +60,7 @@ class _StreamReader(object):
self._text, self.encoding = obj.body, obj.encoding
else:
self._text, self.encoding = obj, 'utf-8'
- self._is_unicode = isinstance(self._text, six.text_type)
+ self._is_unicode = isinstance(self._text, str)
def read(self, n=65535):
self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
- expected_types = (Response, six.text_type, six.binary_type)
+ expected_types = (Response, str, bytes)
assert isinstance(obj, expected_types), \
"obj must be %s, not %s" % (
" or ".join(t.__name__ for t in expected_types),
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
return obj.text
else:
return obj.body.decode('utf-8')
- elif isinstance(obj, six.text_type):
+ elif isinstance(obj, str):
return obj if unicode else obj.encode('utf-8')
else:
return obj.decode('utf-8') if unicode else obj
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index b74f34451..9955fb1e7 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -6,14 +6,13 @@ from contextlib import contextmanager
from importlib import import_module
from pkgutil import iter_modules
-import six
from w3lib.html import replace_entities
from scrapy.utils.python import flatten, to_unicode
from scrapy.item import BaseItem
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
def arg_to_iter(arg):
@@ -83,7 +82,7 @@ def extract_regex(regex, text, encoding='utf-8'):
* if the regex doesn't contain any group the entire regex matching is returned
"""
- if isinstance(regex, six.string_types):
+ if isinstance(regex, str):
regex = re.compile(regex, re.UNICODE)
try:
@@ -92,7 +91,7 @@ def extract_regex(regex, text, encoding='utf-8'):
strings = regex.findall(text) # full regex or numbered groups
strings = flatten(strings)
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py
index f87d5a803..7a7aec9be 100644
--- a/scrapy/utils/ossignal.py
+++ b/scrapy/utils/ossignal.py
@@ -1,5 +1,3 @@
-
-from __future__ import absolute_import
import signal
from twisted.internet import reactor
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index 1cbda141a..f28c2eaa1 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,5 +1,5 @@
import os
-from six.moves import cPickle as pickle
+import pickle
import warnings
from importlib import import_module
@@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
ENVVAR = 'SCRAPY_SETTINGS_MODULE'
DATADIR_CFG_SECTION = 'datadir'
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 138e86d37..8d829c5a5 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -7,7 +7,6 @@ import re
import inspect
import weakref
import errno
-import six
from functools import partial, wraps
from itertools import chain
import sys
@@ -65,10 +64,10 @@ def is_listlike(x):
True
>>> is_listlike((x for x in range(3)))
True
- >>> is_listlike(six.moves.xrange(5))
+ >>> is_listlike(range(5))
True
"""
- return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+ return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
def unique(list_, key=lambda x: x):
@@ -87,9 +86,9 @@ def unique(list_, key=lambda x: x):
def to_unicode(text, encoding=None, errors='strict'):
"""Return the unicode representation of a bytes object ``text``. If
``text`` is already an unicode object, return it as-is."""
- if isinstance(text, six.text_type):
+ if isinstance(text, str):
return text
- if not isinstance(text, (bytes, six.text_type)):
+ if not isinstance(text, (bytes, str)):
raise TypeError('to_unicode must receive a bytes or str '
'object, got %s' % type(text).__name__)
if encoding is None:
@@ -102,7 +101,7 @@ def to_bytes(text, encoding=None, errors='strict'):
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
- if not isinstance(text, six.string_types):
+ if not isinstance(text, str):
raise TypeError('to_bytes must receive a str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
@@ -138,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
yield (text[offset:], offset)
yield (text, 0)
- if isinstance(pattern, six.string_types):
+ if isinstance(pattern, str):
pattern = re.compile(pattern)
for chunk, offset in _chunk_iter():
@@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
return new_method
-_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
_BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
@@ -301,10 +300,10 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
dict or a list of tuples, like any dict ``__init__`` method supports.
"""
d = {}
- for k, v in six.iteritems(dict(dct_or_tuples)):
- k = k.encode(encoding) if isinstance(k, six.text_type) else k
+ for k, v in dict(dct_or_tuples).items():
+ k = k.encode(encoding) if isinstance(k, str) else k
if not keys_only:
- v = v.encode(encoding) if isinstance(v, six.text_type) else v
+ v = v.encode(encoding) if isinstance(v, str) else v
d[k] = v
return d
@@ -346,7 +345,7 @@ def without_none_values(iterable):
value ``None`` have been removed.
"""
try:
- return {k: v for k, v in six.iteritems(iterable) if v is not None}
+ return {k: v for k, v in iterable.items() if v is not None}
except AttributeError:
return type(iterable)((v for v in iterable if v is not None))
diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py
index 495564ac0..749bbc387 100644
--- a/scrapy/utils/reqser.py
+++ b/scrapy/utils/reqser.py
@@ -1,8 +1,6 @@
"""
Helper functions for serializing (and deserializing) requests.
"""
-import six
-
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from scrapy.utils.misc import load_object
@@ -87,12 +85,12 @@ def _mangle_private_name(obj, func, name):
def _find_method(obj, func):
if obj:
try:
- func_self = six.get_method_self(func)
+ func_self = func.__self__
except AttributeError: # func has no __self__
pass
else:
if func_self is obj:
- name = six.get_method_function(func).__name__
+ name = func.__func__.__name__
if _is_private_method(name):
return _mangle_private_name(obj, func, name)
return name
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 0fce5a2e1..356753ab5 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -3,16 +3,15 @@ This module provides some useful functions for working with
scrapy.http.Request objects
"""
-from __future__ import print_function
import hashlib
import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_unicode
-
from w3lib.url import canonicalize_url
+
from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
_fingerprint_cache = weakref.WeakKeyDictionary()
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 4742b3e13..2f10cf4de 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
SitemapSpider, its API is subject to change without notice.
"""
+from urllib.parse import urljoin
+
import lxml.etree
-from six.moves.urllib.parse import urljoin
class Sitemap(object):
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index bf4973fbf..4061d1ea3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -1,11 +1,10 @@
import logging
import inspect
-import six
-
from scrapy.spiders import Spider
from scrapy.utils.misc import arg_to_iter
+
logger = logging.getLogger(__name__)
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
# singleton in scrapy.spider.spiders
from scrapy.spiders import Spider
- for obj in six.itervalues(vars(module)):
+ for obj in vars(module).values():
if inspect.isclass(obj) and \
issubclass(obj, Spider) and \
obj.__module__ == module.__name__ and \
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 9754366df..307c25352 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,7 +2,6 @@
This module contains some assorted functions used in tests
"""
-from __future__ import absolute_import
import os
from importlib import import_module
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index f268e91ff..0f15cf60a 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import
import sys
import os
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index e50a989b3..6f5c21624 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -1,5 +1,4 @@
-from __future__ import print_function
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
from twisted.internet import reactor
from twisted.web import server, resource, static, util
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index eed14c5a1..4842b95df 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -9,12 +9,10 @@ and no performance penalty at all when disabled (as object_ref becomes just an
alias to object in that case).
"""
-from __future__ import print_function
import weakref
from time import time
from operator import itemgetter
from collections import defaultdict
-import six
NoneType = type(None)
@@ -37,13 +35,13 @@ def format_live_refs(ignore=NoneType):
"""Return a tabular representation of tracked objects"""
s = "Live References\n\n"
now = time()
- for cls, wdict in sorted(six.iteritems(live_refs),
+ for cls, wdict in sorted(live_refs.items(),
key=lambda x: x[0].__name__):
if not wdict:
continue
if issubclass(cls, ignore):
continue
- oldest = min(six.itervalues(wdict))
+ oldest = min(wdict.values())
s += "%-30s %6d oldest: %ds ago\n" % (
cls.__name__, len(wdict), now - oldest
)
@@ -57,15 +55,15 @@ def print_live_refs(*a, **kw):
def get_oldest(class_name):
"""Get the oldest object for a specific class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
if not wdict:
break
- return min(six.iteritems(wdict), key=itemgetter(1))[0]
+ return min(wdict.items(), key=itemgetter(1))[0]
def iter_all(class_name):
"""Iterate over all objects of the same class by its class name"""
- for cls, wdict in six.iteritems(live_refs):
+ for cls, wdict in live_refs.items():
if cls.__name__ == class_name:
- return six.iterkeys(wdict)
+ return wdict.keys()
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 2c7b324a1..c9abb12d5 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
"""
import posixpath
import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
# scrapy.utils.url was moved to w3lib.url and import * ensures this
# move doesn't break old code
diff --git a/setup.py b/setup.py
index 8f5f14f0d..85d797f88 100644
--- a/setup.py
+++ b/setup.py
@@ -72,7 +72,6 @@ setup(
'pyOpenSSL>=16.2.0',
'queuelib>=1.4.2',
'service_identity>=16.0.0',
- 'six>=1.10.0',
'w3lib>=1.17.0',
'zope.interface>=4.1.3',
'protego>=0.1.15',
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
new file mode 100644
index 000000000..5f6f1ae30
--- /dev/null
+++ b/tests/CrawlerProcess/simple.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/mockserver.py b/tests/mockserver.py
index d09fbc171..ae249e55a 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -3,9 +3,9 @@ import os
import random
import sys
from subprocess import Popen, PIPE
+from urllib.parse import urlencode
from OpenSSL import SSL
-from six.moves.urllib.parse import urlencode
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index 2ac434f41..d97c4b8ee 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,7 +1,7 @@
# Tests requirements
jmespath
mitmproxy; python_version >= '3.6'
-mitmproxy==3.0.4; python_version < '3.6'
+mitmproxy<4.0.0; python_version < '3.6'
pytest
pytest-cov
pytest-twisted >= 1.11
diff --git a/tests/spiders.py b/tests/spiders.py
index 2487ecc22..981bd2eb8 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking
"""
import time
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
from scrapy.spiders import Spider
from scrapy.http import Request
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 56cfe642a..909ea90e0 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -1,13 +1,12 @@
-from io import StringIO
import json
import os
import pstats
import shutil
-import six
-from subprocess import Popen, PIPE
import sys
import tempfile
import unittest
+from io import StringIO
+from subprocess import Popen, PIPE
from scrapy.utils.test import get_testenv
@@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase):
for char in ("'", "<", ">", 'u"'):
settingsstr = settingsstr.replace(char, '"')
settingsdict = json.loads(settingsstr)
- six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
+ self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
self.assertEqual(200, settingsdict[EXT_PATH])
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index b2e358700..582e3d052 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -1,6 +1,5 @@
from unittest import TextTestResult
-from six import get_unbound_function
from twisted.internet import defer
from twisted.python import failure
from twisted.trial import unittest
@@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
with MockServer() as mockserver:
contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
- get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
- get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
+ TestSameUrlSpider.parse_first.__doc__ = contract_doc
+ TestSameUrlSpider.parse_second.__doc__ = contract_doc
crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
yield crawler.crawl()
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 3fc13eeb7..3307899b7 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -140,7 +140,7 @@ class CrawlTestCase(TestCase):
def test_unbounded_response(self):
# Completeness of responses without Content-Length or Transfer-Encoding
# can not be determined, we treat them as valid but flagged as "partial"
- from six.moves.urllib.parse import urlencode
+ from urllib.parse import urlencode
query = urlencode({'raw': '''\
HTTP/1.1 200 OK
Server: Apache-Coyote/1.1
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 3ac45ca1d..9410b0e7a 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,7 @@
import logging
+import os
+import subprocess
+import sys
import warnings
from pytest import raises, mark
@@ -16,6 +19,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
class BaseCrawlerTest(unittest.TestCase):
@@ -281,3 +285,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
with LogCapture() as log:
yield runner.crawl(AsyncioSpider)
log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+ def run_script(self, script_name):
+ script_path = os.path.join(self.script_dir, script_name)
+ args = (sys.executable, script_path)
+ p = subprocess.Popen(args, env=get_testenv(),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ return stderr.decode('utf-8')
+
+ def test_simple(self):
+ log = self.run_script('simple.py')
+ self.assertIn('Spider closed (finished)', log)
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 8266bf35f..a1645ed96 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -1,6 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
-
from unittest import mock
from twisted.internet import reactor, error
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 30150391a..002c4e6bb 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -10,9 +10,10 @@ module with the ``runserver`` argument::
python test_engine.py runserver
"""
-from __future__ import print_function
-import sys, os, re
-from six.moves.urllib.parse import urlparse
+import os
+import re
+import sys
+from urllib.parse import urlparse
from twisted.internet import reactor, defer
from twisted.web import server, static, util
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 0046c5666..5d1f5c182 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,15 +1,13 @@
-from __future__ import absolute_import
import re
import json
import marshal
+import pickle
import tempfile
import unittest
from io import BytesIO
from datetime import datetime
-from six.moves import cPickle as pickle
import lxml.etree
-import six
from scrapy.item import Item, Field
from scrapy.utils.python import to_unicode
@@ -80,7 +78,7 @@ class BaseItemExporterTest(unittest.TestCase):
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
_, name = list(ie._get_serialized_fields(self.i))[0]
- assert isinstance(name, six.text_type)
+ assert isinstance(name, str)
self.assertEqual(name, u'John\xa3')
def test_field_custom_serializer(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index ce3c4f059..2ca57c19d 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,15 +1,15 @@
-from __future__ import absolute_import
import os
import csv
import json
import warnings
-from io import BytesIO
import tempfile
import shutil
import string
+from io import BytesIO
+from pathlib import Path
from unittest import mock
-from six.moves.urllib.parse import urljoin, urlparse, quote
-from six.moves.urllib.request import pathname2url
+from urllib.parse import urljoin, urlparse, quote
+from urllib.request import pathname2url
from zope.interface.verify import verifyObject
from twisted.trial import unittest
@@ -28,8 +28,6 @@ from scrapy.extensions.feedexport import (
from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
from scrapy.utils.python import to_unicode
-from pathlib import Path
-
class FileFeedStorageTest(unittest.TestCase):
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 0a9ed500a..45ddb42ba 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
from scrapy.http import Request, Response
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index c83cf3b66..cf3fc8496 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -86,9 +86,6 @@ class HeadersTest(unittest.TestCase):
self.assertSortedEqual(h.items(),
[(b'X-Forwarded-For', [b'ip1', b'ip2']),
(b'Content-Type', [b'text/html'])])
- self.assertSortedEqual(h.iteritems(),
- [(b'X-Forwarded-For', [b'ip1', b'ip2']),
- (b'Content-Type', [b'text/html'])])
self.assertSortedEqual(h.values(), [b'ip2', b'text/html'])
def test_update(self):
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 9df6ff67b..a98aa1e6f 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,12 +1,10 @@
import unittest
import re
import json
-from unittest import mock
-from urllib.parse import unquote_to_bytes
+import xmlrpc.client
import warnings
-
-from six.moves import xmlrpc_client as xmlrpclib
-from six.moves.urllib.parse import urlparse, parse_qs
+from unittest import mock
+from urllib.parse import parse_qs, unquote_to_bytes, urlparse
from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
from scrapy.utils.python import to_bytes, to_unicode
@@ -64,7 +62,7 @@ class RequestTest(unittest.TestCase):
# headers must not be unicode
h = Headers({'key1': u'val1', u'key2': 'val2'})
h[u'newkey'] = u'newval'
- for k, v in h.iteritems():
+ for k, v in h.items():
self.assertIsInstance(k, bytes)
for s in v:
self.assertIsInstance(s, bytes)
@@ -1220,7 +1218,7 @@ class XmlRpcRequestTest(RequestTest):
r = self.request_class('http://scrapytest.org/rpc2', **kwargs)
self.assertEqual(r.headers[b'Content-Type'], b'text/xml')
self.assertEqual(r.body,
- to_bytes(xmlrpclib.dumps(**kwargs),
+ to_bytes(xmlrpc.client.dumps(**kwargs),
encoding=kwargs.get('encoding', 'utf-8')))
self.assertEqual(r.method, 'POST')
self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8'))
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 883c943da..79bb745cc 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
import unittest
-import six
from w3lib.encoding import resolve_encoding
from scrapy.http import (Request, Response, TextResponse, HtmlResponse,
@@ -102,7 +101,7 @@ class BaseResponseTest(unittest.TestCase):
self.assertEqual(r4.flags, [])
def _assert_response_values(self, response, encoding, body):
- if isinstance(body, six.text_type):
+ if isinstance(body, str):
body_unicode = body
body_bytes = body.encode(encoding)
else:
@@ -110,7 +109,7 @@ class BaseResponseTest(unittest.TestCase):
body_bytes = body
assert isinstance(response.body, bytes)
- assert isinstance(response.text, six.text_type)
+ assert isinstance(response.text, str)
self._assert_response_encoding(response, encoding)
self.assertEqual(response.body, body_bytes)
self.assertEqual(response.body_as_unicode(), body_unicode)
@@ -220,11 +219,11 @@ class TextResponseTest(BaseResponseTest):
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
# check body_as_unicode
- self.assertTrue(isinstance(r1.body_as_unicode(), six.text_type))
+ self.assertTrue(isinstance(r1.body_as_unicode(), str))
self.assertEqual(r1.body_as_unicode(), unicode_string)
# check response.text
- self.assertTrue(isinstance(r1.text, six.text_type))
+ self.assertTrue(isinstance(r1.text, str))
self.assertEqual(r1.text, unicode_string)
def test_encoding(self):
diff --git a/tests/test_item.py b/tests/test_item.py
index 49117ef04..30463a0f5 100644
--- a/tests/test_item.py
+++ b/tests/test_item.py
@@ -3,8 +3,6 @@ import unittest
from unittest import mock
from warnings import catch_warnings
-import six
-
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import ABCMeta, DictItem, Field, Item, ItemMeta
@@ -302,7 +300,7 @@ class ItemMetaTest(unittest.TestCase):
class ItemMetaClassCellRegression(unittest.TestCase):
def test_item_meta_classcell_regression(self):
- class MyItem(six.with_metaclass(ItemMeta, Item)):
+ class MyItem(Item, metaclass=ItemMeta):
def __init__(self, *args, **kwargs):
# This call to super() trigger the __classcell__ propagation
# requirement. When not done properly raises an error:
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 6bfc31dbf..579a85ff6 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,8 +1,6 @@
from functools import partial
import unittest
-import six
-
from scrapy.http import HtmlResponse
from scrapy.item import Item, Field
from scrapy.loader import ItemLoader
@@ -157,7 +155,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_get_value(self):
il = NameItemLoader()
- self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
+ self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), str.upper))
self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
@@ -258,7 +256,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_extend_custom_input_processors(self):
class ChildItemLoader(TestItemLoader):
- name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
+ name_in = MapCompose(TestItemLoader.name_in, str.swapcase)
il = ChildItemLoader()
il.add_value('name', u'marta')
@@ -266,7 +264,7 @@ class BasicItemLoaderTest(unittest.TestCase):
def test_extend_default_input_processors(self):
class ChildDefaultedItemLoader(DefaultedItemLoader):
- name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
+ name_in = MapCompose(DefaultedItemLoader.default_input_processor, str.swapcase)
il = ChildDefaultedItemLoader()
il.add_value('name', u'marta')
@@ -689,7 +687,7 @@ class ProcessorsTest(unittest.TestCase):
self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
self.assertEqual(proc(['hello', 'world']), u'hello world')
- self.assertIsInstance(proc(['hello', 'world']), six.text_type)
+ self.assertIsInstance(proc(['hello', 'world']), str)
def test_compose(self):
proc = Compose(lambda v: v[0], str.upper)
@@ -704,12 +702,12 @@ class ProcessorsTest(unittest.TestCase):
def test_mapcompose(self):
def filter_world(x):
return None if x == 'world' else x
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(None), [])
- proc = MapCompose(filter_world, six.text_type.upper)
+ proc = MapCompose(filter_world, str.upper)
self.assertRaises(ValueError, proc, [1])
proc = MapCompose(filter_world, lambda x: x + 1)
self.assertRaises(ValueError, proc, 'hello')
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index d0b23a8c4..7d8c6ec7f 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -3,7 +3,6 @@ import unittest
from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase as TwistedTestCase
-import six
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import DropItem
@@ -60,7 +59,7 @@ class LogFormatterTestCase(unittest.TestCase):
logkws = self.formatter.dropped(item, exception, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
- assert all(isinstance(x, six.text_type) for x in lines)
+ assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
def test_error(self):
@@ -80,7 +79,7 @@ class LogFormatterTestCase(unittest.TestCase):
logkws = self.formatter.scraped(item, response, self.spider)
logline = logkws['msg'] % logkws['args']
lines = logline.splitlines()
- assert all(isinstance(x, six.text_type) for x in lines)
+ assert all(isinstance(x, str) for x in lines)
self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index bd40e4103..52f2b554e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,11 +1,11 @@
import os
import random
import time
+from io import BytesIO
from tempfile import mkdtemp
from shutil import rmtree
from unittest import mock
-from six.moves.urllib.parse import urlparse
-from six import BytesIO
+from urllib.parse import urlparse
from twisted.trial import unittest
from twisted.internet import defer
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ad958e25f..0d23f51cc 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,3 @@
-from __future__ import print_function
-
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 277455751..5f68cd448 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -1,20 +1,20 @@
import json
import os
import re
-from subprocess import Popen, PIPE
import sys
+from subprocess import Popen, PIPE
+from urllib.parse import urlsplit, urlunsplit
import pytest
-from six.moves.urllib.parse import urlsplit, urlunsplit
from testfixtures import LogCapture
-
from twisted.internet import defer
from twisted.trial.unittest import TestCase
-from scrapy.utils.test import get_crawler
from scrapy.http import Request
-from tests.spiders import SimpleSpider, SingleRequestSpider
+from scrapy.utils.test import get_crawler
+
from tests.mockserver import MockServer
+from tests.spiders import SimpleSpider, SingleRequestSpider
class MitmProxy:
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 32e65bed5..fda44653a 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,4 +1,3 @@
-import six
import unittest
from unittest import mock
@@ -10,7 +9,7 @@ from . import default_settings
class SettingsGlobalFuncsTest(unittest.TestCase):
def test_get_settings_priority(self):
- for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+ for prio_str, prio_num in SETTINGS_PRIORITIES.items():
self.assertEqual(get_settings_priority(prio_str), prio_num)
self.assertEqual(get_settings_priority(99), 99)
@@ -43,14 +42,14 @@ class SettingsAttributeTest(unittest.TestCase):
new_dict = {'three': 11, 'four': 21}
attribute.set(new_dict, 10)
self.assertIsInstance(attribute.value, BaseSettings)
- six.assertCountEqual(self, attribute.value, new_dict)
- six.assertCountEqual(self, original_settings, original_dict)
+ self.assertCountEqual(attribute.value, new_dict)
+ self.assertCountEqual(original_settings, original_dict)
new_settings = BaseSettings({'five': 12}, 0)
attribute.set(new_settings, 0) # Insufficient priority
- six.assertCountEqual(self, attribute.value, new_dict)
+ self.assertCountEqual(attribute.value, new_dict)
attribute.set(new_settings, 10)
- six.assertCountEqual(self, attribute.value, new_settings)
+ self.assertCountEqual(attribute.value, new_settings)
def test_repr(self):
self.assertEqual(repr(self.attribute),
@@ -148,10 +147,10 @@ class BaseSettingsTest(unittest.TestCase):
self.settings.setmodule(
'tests.test_settings.default_settings', 10)
- self.assertCountEqual(six.iterkeys(self.settings.attributes),
- six.iterkeys(ctrl_attributes))
+ self.assertCountEqual(self.settings.attributes.keys(),
+ ctrl_attributes.keys())
- for key in six.iterkeys(ctrl_attributes):
+ for key in ctrl_attributes.keys():
attr = self.settings.attributes[key]
ctrl_attr = ctrl_attributes[key]
self.assertEqual(attr.value, ctrl_attr.value)
@@ -227,7 +226,7 @@ class BaseSettingsTest(unittest.TestCase):
}
settings = self.settings
settings.attributes = {key: SettingsAttribute(value, 0) for key, value
- in six.iteritems(test_configuration)}
+ in test_configuration.items()}
self.assertTrue(settings.getbool('TEST_ENABLED1'))
self.assertTrue(settings.getbool('TEST_ENABLED2'))
@@ -276,9 +275,8 @@ class BaseSettingsTest(unittest.TestCase):
'TEST': BaseSettings({1: 10, 3: 30}, 'default'),
'HASNOBASE': BaseSettings({3: 3000}, 'default')})
s['TEST'].set(2, 200, 'cmdline')
- six.assertCountEqual(self, s.getwithbase('TEST'),
- {1: 1, 2: 200, 3: 30})
- six.assertCountEqual(self, s.getwithbase('HASNOBASE'), s['HASNOBASE'])
+ self.assertCountEqual(s.getwithbase('TEST'), {1: 1, 2: 200, 3: 30})
+ self.assertCountEqual(s.getwithbase('HASNOBASE'), s['HASNOBASE'])
self.assertEqual(s.getwithbase('NONEXISTENT'), {})
def test_maxpriority(self):
diff --git a/tests/test_spider.py b/tests/test_spider.py
index aa43e3b3a..317a27076 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -384,6 +384,14 @@ class CrawlSpiderTest(SpiderTest):
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
+ def test_start_url(self):
+ spider = self.spider_class("example.com")
+ spider.start_url = 'https://www.example.com'
+
+ with self.assertRaisesRegex(AttributeError,
+ r'^Crawling could not start.*$'):
+ list(spider.start_requests())
+
class SitemapSpiderTest(SpiderTest):
diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py
index b97d9b675..992e60be2 100644
--- a/tests/test_spidermiddleware_offsite.py
+++ b/tests/test_spidermiddleware_offsite.py
@@ -1,13 +1,11 @@
from unittest import TestCase
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
+import warnings
from scrapy.http import Response, Request
from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware
-from scrapy.spidermiddlewares.offsite import URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
from scrapy.utils.test import get_crawler
-import warnings
class TestOffsiteMiddleware(TestCase):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 2be6a1cd5..ecec6135d 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from unittest import TestCase
import warnings
diff --git a/tests/test_toplevel.py b/tests/test_toplevel.py
index 91bbe43bc..fdc5df166 100644
--- a/tests/test_toplevel.py
+++ b/tests/test_toplevel.py
@@ -1,12 +1,12 @@
from unittest import TestCase
-import six
+
import scrapy
class ToplevelTestCase(TestCase):
def test_version(self):
- self.assertIs(type(scrapy.__version__), six.text_type)
+ self.assertIs(type(scrapy.__version__), str)
def test_version_info(self):
self.assertIs(type(scrapy.version_info), tuple)
diff --git a/tests/test_urlparse_monkeypatches.py b/tests/test_urlparse_monkeypatches.py
index 22e39821c..bea0cf3e5 100644
--- a/tests/test_urlparse_monkeypatches.py
+++ b/tests/test_urlparse_monkeypatches.py
@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
import unittest
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index c5655df7e..50e1bfd5f 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,7 +1,6 @@
import unittest
import warnings
-from six import assertRaisesRegex
from w3lib.http import basic_auth_header
from scrapy import Request
@@ -177,8 +176,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
def test_too_few_arguments_error(self):
- assertRaisesRegex(
- self,
+ self.assertRaisesRegex(
ValueError,
r"too few arguments|the following arguments are required:\s*url",
lambda: curl_to_request_kwargs("curl"),
@@ -194,8 +192,7 @@ class CurlToRequestKwargsTest(unittest.TestCase):
self.assertEqual(curl_to_request_kwargs(curl_command), expected_result)
# case 2: ignore_unknown_options=False (raise exception):
- assertRaisesRegex(
- self,
+ self.assertRaisesRegex(
ValueError,
"Unrecognized options:.*--bar.*--baz",
lambda: curl_to_request_kwargs(
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index 53228fc6e..38a25778e 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -192,14 +192,6 @@ class SequenceExcludeTest(unittest.TestCase):
self.assertIn(20, d)
self.assertNotIn(15, d)
- def test_six_range(self):
- import six.moves
- seq = six.moves.range(10**3, 10**6)
- d = SequenceExclude(seq)
- self.assertIn(10**2, d)
- self.assertIn(10**7, d)
- self.assertNotIn(10**4, d)
-
def test_range_step(self):
seq = range(10, 20, 3)
d = SequenceExclude(seq)
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index d642ed3ed..49c2befb5 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,8 +5,6 @@ from twisted.python.failure import Failure
from scrapy.utils.defer import mustbe_deferred, process_chain, \
process_chain_both, process_parallel, iter_errback
-from six.moves import xrange
-
class MustbeDeferredTest(unittest.TestCase):
def test_success_function(self):
@@ -92,7 +90,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_good(self):
def itergood():
- for x in xrange(10):
+ for x in range(10):
yield x
errors = []
@@ -102,7 +100,7 @@ class IterErrbackTest(unittest.TestCase):
def test_iter_errback_bad(self):
def iterbad():
- for x in xrange(10):
+ for x in range(10):
if x == 5:
a = 1/0
yield x
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index ce04e7f29..159ef8f25 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import absolute_import
import inspect
import unittest
from unittest import mock
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 2c3965bbc..cf8ad1f23 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,5 +1,5 @@
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Request
from scrapy.utils.httpobj import urlparse_cached
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index f16ef8110..9776dfb2a 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
import os
-import six
+
from twisted.trial import unittest
from scrapy.utils.iterators import csviter, xmliter, _body_or_str, xmliter_lxml
from scrapy.http import XmlResponse, TextResponse, Response
from tests import get_testdata
+
FOOBAR_NL = u"foo\nbar"
@@ -256,8 +257,8 @@ class UtilsCsvTestCase(unittest.TestCase):
# explicit type check cuz' we no like stinkin' autocasting! yarrr
for result_row in result:
- self.assertTrue(all((isinstance(k, six.text_type) for k in result_row.keys())))
- self.assertTrue(all((isinstance(v, six.text_type) for v in result_row.values())))
+ self.assertTrue(all((isinstance(k, str) for k in result_row.keys())))
+ self.assertTrue(all((isinstance(v, str) for v in result_row.values())))
def test_csviter_delimiter(self):
body = get_testdata('feeds', 'feed-sample3.csv').replace(b',', b'\t')
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 742e04803..2c23f3616 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,5 +1,4 @@
# -*- coding: utf-8 -*-
-from __future__ import print_function
import sys
import logging
import unittest
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 2d27d4b81..b79e0ac1c 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -1,10 +1,9 @@
-import gc
import functools
+import gc
import operator
+import platform
import unittest
from itertools import count
-import platform
-import six
from warnings import catch_warnings
from scrapy.utils.python import (
@@ -12,6 +11,7 @@ from scrapy.utils.python import (
WeakKeyCache, get_func_args, to_bytes, to_unicode,
without_none_values, MutableChain)
+
__doctests__ = ['scrapy.utils.python']
@@ -205,12 +205,12 @@ class UtilsPythonTestCase(unittest.TestCase):
if platform.python_implementation() == 'CPython':
# TODO: how do we fix this to return the actual argument names?
- self.assertEqual(get_func_args(six.text_type.split), [])
+ self.assertEqual(get_func_args(str.split), [])
self.assertEqual(get_func_args(" ".join), [])
self.assertEqual(get_func_args(operator.itemgetter(2)), [])
else:
self.assertEqual(
- get_func_args(six.text_type.split, stripself=True), ['sep', 'maxsplit'])
+ get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
self.assertEqual(
get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 3da95b95a..3e664fc74 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
import unittest
from scrapy.http import Request
from scrapy.utils.request import request_fingerprint, _fingerprint_cache, \
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index bea4dade3..6ebf290c0 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,12 +1,13 @@
import os
import unittest
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
from scrapy.http import Response, TextResponse, HtmlResponse
from scrapy.utils.python import to_bytes
from scrapy.utils.response import (response_httprepr, open_in_browser,
get_meta_refresh, get_base_url, response_status_message)
+
__doctests__ = ['scrapy.utils.response']
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 480a717e7..16e02f919 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,6 +1,7 @@
-import six
import unittest
+from io import StringIO
from unittest import mock
+
from scrapy.utils import trackref
@@ -38,12 +39,12 @@ Live References
Bar 1 oldest: 0s ago
''')
- @mock.patch('sys.stdout', new_callable=six.StringIO)
+ @mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_empty(self, stdout):
trackref.print_live_refs()
self.assertEqual(stdout.getvalue(), 'Live References\n\n\n')
- @mock.patch('sys.stdout', new_callable=six.StringIO)
+ @mock.patch('sys.stdout', new_callable=StringIO)
def test_print_live_refs_with_objects(self, stdout):
o1 = Foo() # NOQA
trackref.print_live_refs()
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index c7bcaf88b..21e9a056a 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -5,6 +5,7 @@ from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
add_http_if_no_scheme, guess_scheme, strip_url)
+
__doctests__ = ['scrapy.utils.url']
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 7b015ff8d..746367b41 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -3,7 +3,6 @@ from twisted.internet import defer
Tests borrowed from the twisted.web.client tests.
"""
import os
-import six
import shutil
import OpenSSL.SSL
@@ -298,7 +297,7 @@ class WebClientTestCase(unittest.TestCase):
def cleanup(passthrough):
# Clean up the server which is hanging around not doing
# anything.
- connected = list(six.iterkeys(self.wrapper.protocols))
+ connected = list(self.wrapper.protocols.keys())
# There might be nothing here if the server managed to already see
# that the connection was lost.
if connected: