Merge remote-tracking branch 'origin/master' into asyncio-base

.bandit.yml

@@ -1,13 +1,15 @@
 skips:
 - B101
 - B105
+- B301
 - B303
 - B306
 - B307
 - B311
 - B320
 - B321
-- B402
+- B402  # https://github.com/scrapy/scrapy/issues/4180
+- B403
 - B404
 - B406
 - B410

conftest.py

@@ -1,12 +1,19 @@
+from pathlib import Path
+
 import pytest
 
 
+def _py_files(folder):
+    return (str(p) for p in Path(folder).rglob('*.py'))
+
+
 collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testsite.py",
+    # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+    *_py_files("tests/CrawlerProcess")
 ]
 
-
 for line in open('tests/ignores.txt'):
     file_path = line.strip()
     if file_path and file_path[0] != '#':

docs/intro/overview.rst

@@ -34,8 +34,8 @@ http://quotes.toscrape.com, following the pagination::
         def parse(self, response):
             for quote in response.css('div.quote'):
                 yield {
-                    'text': quote.css('span.text::text').get(),
                     'author': quote.xpath('span/small/text()').get(),
+                    'text': quote.css('span.text::text').get(),
                 }
 
             next_page = response.css('li.next a::attr("href")').get()

extras/qps-bench-server.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 from time import time
 from collections import deque
 from twisted.web.server import Site, NOT_DONE_YET

requirements-py3.txt

@@ -2,7 +2,6 @@ parsel>=1.5.0
 PyDispatcher>=2.0.5
 Twisted>=17.9.0
 w3lib>=1.17.0
-protego>=0.1.15
 
 pyOpenSSL>=16.2.0  # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
 queuelib>=1.4.2    # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
@@ -14,5 +13,4 @@ cryptography>=2.0  # Earlier versions would fail to install
 cssselect>=0.9.1
 lxml>=3.5.0
 service_identity>=16.0.0
-six>=1.10.0
 zope.interface>=4.1.3

scrapy/__init__.py

@@ -24,7 +24,7 @@ warnings.filterwarnings('ignore', category=DeprecationWarning, module='twisted')
 del warnings
 
 # Apply monkey patches to fix issues in external libraries
-from . import _monkeypatches
+from scrapy import _monkeypatches
 del _monkeypatches
 
 from twisted import version as _txv

scrapy/_monkeypatches.py

@@ -1,4 +1,4 @@
-from six.moves import copyreg
+import copyreg
 
 
 # Undo what Twisted's perspective broker adds to pickle register

scrapy/cmdline.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys
 import os
 import optparse

scrapy/commands/bench.py

@@ -1,8 +1,7 @@
 import sys
 import time
 import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 import scrapy
 from scrapy.commands import ScrapyCommand

scrapy/commands/fetch.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import sys
 from w3lib.url import is_url
 

scrapy/commands/genspider.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import shutil
 import string

scrapy/commands/list.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 from scrapy.commands import ScrapyCommand
 
 

scrapy/commands/parse.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 import logging
 

scrapy/commands/settings.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import json
 
 from scrapy.commands import ScrapyCommand

scrapy/commands/startproject.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import re
 import os
 import string

scrapy/commands/version.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.versions import scrapy_components_versions

scrapy/contracts/default.py

@@ -4,7 +4,7 @@ from scrapy.item import BaseItem
 from scrapy.http import Request
 from scrapy.exceptions import ContractFail
 
-from . import Contract
+from scrapy.contracts import Contract
 
 
 # contracts

scrapy/core/downloader/__init__.py

@@ -3,15 +3,14 @@ from time import time
 from datetime import datetime
 from collections import deque
 
-import six
 from twisted.internet import reactor, defer, task
 
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.resolver import dnscache
 from scrapy import signals
-from .middleware import DownloaderMiddlewareManager
-from .handlers import DownloadHandlers
+from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.core.downloader.handlers import DownloadHandlers
 
 
 class Slot(object):
@@ -188,7 +187,7 @@ class Downloader(object):
 
     def close(self):
         self._slot_gc_loop.stop()
-        for slot in six.itervalues(self.slots):
+        for slot in self.slots.values():
             slot.close()
 
     def _slot_gc(self, age=60):

scrapy/core/downloader/handlers/__init__.py

@@ -1,8 +1,9 @@
 """Download handlers for different schemes"""
 
 import logging
+
 from twisted.internet import defer
-import six
+
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
         self._notconfigured = {}  # remembers failed handlers
         handlers = without_none_values(
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
-        for scheme, clspath in six.iteritems(handlers):
+        for scheme, clspath in handlers.items():
             self._schemes[scheme] = clspath
             self._load_handler(scheme, skip_lazy=True)
 

scrapy/core/downloader/handlers/ftp.py

@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed

scrapy/core/downloader/handlers/http.py

@@ -1,2 +1,4 @@
-from .http10 import HTTP10DownloadHandler
-from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
+from scrapy.core.downloader.handlers.http11 import (
+    HTTP11DownloadHandler as HTTPDownloadHandler,
+)

scrapy/core/downloader/handlers/http11.py

@@ -2,10 +2,10 @@
 
 import re
 import logging
+import warnings
 from io import BytesIO
 from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
 
 from zope.interface import implementer
 from twisted.internet import defer, reactor, protocol

scrapy/core/downloader/handlers/s3.py

@@ -1,9 +1,9 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.boto import is_botocore
-from .http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 
 
 def _get_boto_connection():

scrapy/core/downloader/middleware.py

@@ -3,8 +3,6 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-import six
-
 from twisted.internet import defer
 
 from scrapy.exceptions import _InvalidOutput
@@ -38,7 +36,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, response.__class__.__name__))
+                                         (method.__self__.__class__.__name__, response.__class__.__name__))
                 if response:
                     defer.returnValue(response)
             defer.returnValue((yield download_func(request=request, spider=spider)))
@@ -53,7 +51,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, response=response, spider=spider)
                 if not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, type(response)))
+                                         (method.__self__.__class__.__name__, type(response)))
                 if isinstance(response, Request):
                     defer.returnValue(response)
             defer.returnValue(response)
@@ -65,7 +63,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 response = yield method(request=request, exception=exception, spider=spider)
                 if response is not None and not isinstance(response, (Response, Request)):
                     raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                                         (six.get_method_self(method).__class__.__name__, type(response)))
+                                         (method.__self__.__class__.__name__, type(response)))
                 if response:
                     defer.returnValue(response)
             defer.returnValue(_failure)

scrapy/core/downloader/webclient.py

@@ -1,5 +1,5 @@
 from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
 
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient

scrapy/core/spidermw.py

@@ -5,7 +5,6 @@ See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import chain, islice
 
-import six
 from twisted.python.failure import Failure
 from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
@@ -37,8 +36,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     def scrape_response(self, scrape_func, response, request, spider):
         fname = lambda f: '%s.%s' % (
-                six.get_method_self(f).__class__.__name__,
-                six.get_method_function(f).__name__)
+                f.__self__.__class__.__name__,
+                f.__func__.__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:

scrapy/crawler.py

@@ -1,7 +1,6 @@
-import pprint
-import six
-import signal
 import logging
+import pprint
+import signal
 import warnings
 
 from twisted.internet import reactor, defer
@@ -22,6 +21,7 @@ from scrapy.utils.log import (
     get_scrapy_root_handler, install_scrapy_root_handler)
 from scrapy import signals
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -206,7 +206,7 @@ class CrawlerRunner(object):
         return self._create_crawler(crawler_or_spidercls)
 
     def _create_crawler(self, spidercls):
-        if isinstance(spidercls, six.string_types):
+        if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
         return Crawler(spidercls, self.settings)
 

scrapy/downloadermiddlewares/ajaxcrawl.py

@@ -1,9 +1,7 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 import re
 import logging
 
-import six
 from w3lib import html
 
 from scrapy.exceptions import NotConfigured
@@ -67,7 +65,7 @@ class AjaxCrawlMiddleware(object):
 
 
 # XXX: move it to w3lib?
-_ajax_crawlable_re = re.compile(six.u(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'))
+_ajax_crawlable_re = re.compile(r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>')
 
 
 def _has_ajaxcrawlable_meta(text):

scrapy/downloadermiddlewares/cookies.py

@@ -1,13 +1,12 @@
 import logging
 from collections import defaultdict
 
-import six
-
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +81,10 @@ class CookiesMiddleware(object):
 
     def _get_request_cookies(self, jar, request):
         if isinstance(request.cookies, dict):
-            cookie_list = [{'name': k, 'value': v} for k, v in \
-                    six.iteritems(request.cookies)]
+            cookie_list = [
+                {'name': k, 'value': v}
+                for k, v in request.cookies.items()
+            ]
         else:
             cookie_list = request.cookies
 

scrapy/downloadermiddlewares/decompression.py

@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
-from io import BytesIO
-import zipfile
-import tarfile
 import logging
+import tarfile
+import zipfile
+from io import BytesIO
 from tempfile import mktemp
 
-import six
-
 from scrapy.responsetypes import responsetypes
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
         if not response.body:
             return response
 
-        for fmt, func in six.iteritems(self._formats):
+        for fmt, func in self._formats.items():
             new_response = func(response)
             if new_response:
                 logger.debug('Decompressed response with format: %(responsefmt)s',

scrapy/downloadermiddlewares/httpproxy.py

@@ -1,8 +1,6 @@
 import base64
-from urllib.request import _parse_proxy
-
-from six.moves.urllib.parse import unquote, urlunparse
-from six.moves.urllib.request import getproxies, proxy_bypass
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/downloadermiddlewares/redirect.py

@@ -1,5 +1,5 @@
 import logging
-from six.moves.urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin, urlparse
 
 from w3lib.url import safe_url_string
 
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
+
 logger = logging.getLogger(__name__)
 
 

scrapy/dupefilters.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 import os
 import logging
 

scrapy/exporters.py

@@ -6,15 +6,14 @@ import csv
 import io
 import pprint
 import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+import pickle
 from xml.sax.saxutils import XMLGenerator
 
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
@@ -60,9 +59,9 @@ class BaseItemExporter(object):
             include_empty = self.export_empty_fields
         if self.fields_to_export is None:
             if include_empty and not isinstance(item, dict):
-                field_iter = six.iterkeys(item.fields)
+                field_iter = item.fields.keys()
             else:
-                field_iter = six.iterkeys(item)
+                field_iter = item.keys()
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -180,7 +179,7 @@ class XmlItemExporter(BaseItemExporter):
             for value in serialized_value:
                 self._export_xml_field('value', value, depth=depth+1)
             self._beautify_indent(depth=depth)
-        elif isinstance(serialized_value, six.text_type):
+        elif isinstance(serialized_value, str):
             self.xg.characters(serialized_value)
         else:
             self.xg.characters(str(serialized_value))
@@ -319,12 +318,12 @@ class PythonItemExporter(BaseItemExporter):
         if is_listlike(value):
             return [self._serialize_value(v) for v in value]
         encode_func = to_bytes if self.binary else to_unicode
-        if isinstance(value, (six.text_type, bytes)):
+        if isinstance(value, (str, bytes)):
             return encode_func(value, encoding=self.encoding)
         return value
 
     def _serialize_dict(self, value):
-        for key, val in six.iteritems(value):
+        for key, val in value.items():
             key = to_bytes(key) if self.binary else key
             yield key, self._serialize_value(val)
 

scrapy/extensions/feedexport.py

@@ -10,7 +10,7 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-from six.moves.urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote
 from ftplib import FTP
 
 from zope.interface import Interface, implementer

scrapy/extensions/httpcache.py

@@ -1,14 +1,12 @@
-from __future__ import print_function
-
 import gzip
 import logging
 import os
+import pickle
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
 from weakref import WeakKeyDictionary
 
-from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
 
 from scrapy.http import Headers, Response

scrapy/extensions/memdebug.py

@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
 """
 
 import gc
-import six
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
     def spider_closed(self, spider, reason):
         gc.collect()
         self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
-        for cls, wdict in six.iteritems(live_refs):
+        for cls, wdict in live_refs.items():
             if not wdict:
                 continue
             self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)

scrapy/extensions/spiderstate.py

@@ -1,5 +1,5 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured

scrapy/http/cookies.py

@@ -1,7 +1,6 @@
 import time
-from six.moves.http_cookiejar import (
-    CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 

scrapy/http/headers.py

@@ -1,4 +1,3 @@
-import six
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
@@ -19,7 +18,7 @@ class Headers(CaselessDict):
         """Normalize values to bytes"""
         if value is None:
             value = []
-        elif isinstance(value, (six.text_type, bytes)):
+        elif isinstance(value, (str, bytes)):
             value = [value]
         elif not hasattr(value, '__iter__'):
             value = [value]
@@ -29,10 +28,10 @@ class Headers(CaselessDict):
     def _tobytes(self, x):
         if isinstance(x, bytes):
             return x
-        elif isinstance(x, six.text_type):
+        elif isinstance(x, str):
             return x.encode(self.encoding)
         elif isinstance(x, int):
-            return six.text_type(x).encode(self.encoding)
+            return str(x).encode(self.encoding)
         else:
             raise TypeError('Unsupported value type: {}'.format(type(x)))
 
@@ -68,9 +67,6 @@ class Headers(CaselessDict):
         self[key] = lst
 
     def items(self):
-        return list(self.iteritems())
-
-    def iteritems(self):
         return ((k, self.getlist(k)) for k in self.keys())
 
     def values(self):

scrapy/http/request/__init__.py

@@ -4,7 +4,6 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-import six
 from w3lib.url import safe_url_string
 
 from scrapy.http.headers import Headers
@@ -60,7 +59,7 @@ class Request(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if not isinstance(url, six.string_types):
+        if not isinstance(url, str):
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
 
         s = safe_url_string(url, self.encoding)

scrapy/http/request/form.py

@@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
 
 import lxml.html
 from parsel.selector import create_root_node
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
     xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')

scrapy/http/request/rpc.py

@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
 
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args

scrapy/http/response/__init__.py

@@ -4,7 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from scrapy.http.request import Request
 from scrapy.http.headers import Headers

scrapy/http/response/text.py

@@ -5,8 +5,7 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 import parsel
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
@@ -31,14 +30,14 @@ class TextResponse(Response):
         super(TextResponse, self).__init__(*args, **kwargs)
 
     def _set_url(self, url):
-        if isinstance(url, six.text_type):
+        if isinstance(url, str):
             self._url = to_unicode(url, self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
 
     def _set_body(self, body):
         self._body = b''  # used by encoding detection
-        if isinstance(body, six.text_type):
+        if isinstance(body, str):
             if self._encoding is None:
                 raise TypeError('Cannot convert unicode body - %s has no encoding' %
                     type(self).__name__)
@@ -158,7 +157,7 @@ class TextResponse(Response):
 
 def _url_from_selector(sel):
     # type: (parsel.Selector) -> str
-    if isinstance(sel.root, six.string_types):
+    if isinstance(sel.root, str):
         # e.g. ::attr(href) result
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, 'tag'):

scrapy/item.py

@@ -10,8 +10,6 @@ from copy import deepcopy
 from pprint import pformat
 from warnings import warn
 
-import six
-
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.trackref import object_ref
 
@@ -78,7 +76,7 @@ class DictItem(MutableMapping, BaseItem):
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case
-            for k, v in six.iteritems(dict(*args, **kwargs)):
+            for k, v in dict(*args, **kwargs).items():
                 self[k] = v
 
     def __getitem__(self, key):
@@ -130,6 +128,5 @@ class DictItem(MutableMapping, BaseItem):
         return deepcopy(self)
 
 
-@six.add_metaclass(ItemMeta)
-class Item(DictItem):
+class Item(DictItem, metaclass=ItemMeta):
     pass

scrapy/linkextractors/__init__.py

@@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
 For more info see docs/topics/link-extractors.rst
 """
 import re
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 from parsel.csstranslator import HTMLTranslator
 from w3lib.url import canonicalize_url
 
@@ -118,4 +118,4 @@ class FilteringLinkExtractor(object):
 
 
 # Top-level imports
-from .lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor as LinkExtractor  # noqa: F401

scrapy/linkextractors/htmlparser.py

@@ -2,9 +2,8 @@
 HTMLParser-based link extractor
 """
 import warnings
-import six
-from six.moves.html_parser import HTMLParser
-from six.moves.urllib.parse import urljoin
+from html.parser import HTMLParser
+from urllib.parse import urljoin
 
 from w3lib.url import safe_url_string
 from w3lib.html import strip_html5_whitespace
@@ -42,7 +41,7 @@ class HtmlParserLinkExtractor(HTMLParser):
         ret = []
         base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in links:
-            if isinstance(link.url, six.text_type):
+            if isinstance(link.url, str):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)

scrapy/linkextractors/lxmlhtml.py

@@ -1,8 +1,7 @@
 """
 Link extractor based on lxml.html
 """
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
@@ -22,7 +21,7 @@ _collect_string_content = etree.XPath("string()")
 
 
 def _nons(tag):
-    if isinstance(tag, six.string_types):
+    if isinstance(tag, str):
         if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE:
             return tag.split('}')[-1]
     return tag

scrapy/linkextractors/regex.py

@@ -1,10 +1,11 @@
 import re
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
-from .sgml import SgmlLinkExtractor
+from scrapy.linkextractors.sgml import SgmlLinkExtractor
+
 
 linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",

scrapy/linkextractors/sgml.py

@@ -1,9 +1,8 @@
 """
 SGMLParser-based Link extractors
 """
-import six
-from six.moves.urllib.parse import urljoin
 import warnings
+from urllib.parse import urljoin
 from sgmllib import SGMLParser
 
 from w3lib.url import safe_url_string, canonicalize_url
@@ -49,7 +48,7 @@ class BaseSgmlLinkExtractor(SGMLParser):
         if base_url is None:
             base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
         for link in self.links:
-            if isinstance(link.url, six.text_type):
+            if isinstance(link.url, str):
                 link.url = link.url.encode(response_encoding)
             try:
                 link.url = urljoin(base_url, link.url)

scrapy/mail.py

@@ -3,21 +3,21 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
-from io import BytesIO
 import logging
-
-from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-from email.mime.nonmultipart import MIMENonMultipart
 from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from io import BytesIO
 
 from twisted.internet import defer, reactor, ssl
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
+
 logger = logging.getLogger(__name__)
 
 

scrapy/pipelines/files.py

@@ -5,16 +5,14 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
-from io import BytesIO
+import logging
 import mimetypes
 import os
-import os.path
 import time
-import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
 from collections import defaultdict
-import six
+from email.utils import parsedate_tz, mktime_tz
+from io import BytesIO
+from urllib.parse import urlparse
 
 from twisted.internet import defer, threads
 
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -153,14 +152,14 @@ class S3FilesStore(object):
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
-                Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+                Metadata={k: str(v) for k, v in (meta or {}).items()},
                 ACL=self.POLICY,
                 **extra)
         else:
             b = self._get_boto_bucket()
             k = b.new_key(key_name)
             if meta:
-                for metakey, metavalue in six.iteritems(meta):
+                for metakey, metavalue in meta.items():
                     k.set_metadata(metakey, str(metavalue))
             h = self.HEADERS.copy()
             if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
             'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
         })
         extra = {}
-        for key, value in six.iteritems(headers):
+        for key, value in headers.items():
             try:
                 kwarg = mapping[key]
             except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
     def persist_file(self, path, buf, info, meta=None, headers=None):
         blob = self.bucket.blob(self.prefix + path)
         blob.cache_control = self.CACHE_CONTROL
-        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),

scrapy/pipelines/images.py

@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
 import functools
 import hashlib
 from io import BytesIO
-import six
 
 from PIL import Image
 
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
         image, buf = self.convert_image(orig_image)
         yield path, image, buf
 
-        for thumb_id, size in six.iteritems(self.thumbs):
+        for thumb_id, size in self.thumbs.items():
             thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf

scrapy/pipelines/media.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import functools
 import logging
 from collections import defaultdict

scrapy/responsetypes.py

@@ -2,11 +2,9 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
-from __future__ import absolute_import
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
-import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
@@ -37,7 +35,7 @@ class ResponseTypes(object):
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types').decode('utf8')
         self.mimetypes.readfp(StringIO(mimedata))
-        for mimetype, cls in six.iteritems(self.CLASSES):
+        for mimetype, cls in self.CLASSES.items():
             self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype):

scrapy/robotstxt.py

@@ -1,7 +1,6 @@
 import sys
 import logging
 from abc import ABCMeta, abstractmethod
-from six import with_metaclass
 
 from scrapy.utils.python import to_unicode
 
@@ -26,7 +25,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
     return robotstxt_body
 
 
-class RobotParser(with_metaclass(ABCMeta)):
+class RobotParser(metaclass=ABCMeta):
     @classmethod
     @abstractmethod
     def from_crawler(cls, crawler, robotstxt_body):
@@ -56,7 +55,7 @@ class RobotParser(with_metaclass(ABCMeta)):
 
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
-        from six.moves.urllib_robotparser import RobotFileParser
+        from urllib.robotparser import RobotFileParser
         self.spider = spider
         robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp = RobotFileParser()

scrapy/settings/__init__.py

@@ -1,4 +1,3 @@
-import six
 import json
 import copy
 from collections.abc import MutableMapping
@@ -23,7 +22,7 @@ def get_settings_priority(priority):
     :attr:`~scrapy.settings.SETTINGS_PRIORITIES` dictionary and returns its
     numerical value, or directly returns a given numerical priority.
     """
-    if isinstance(priority, six.string_types):
+    if isinstance(priority, str):
         return SETTINGS_PRIORITIES[priority]
     else:
         return priority
@@ -173,7 +172,7 @@ class BaseSettings(MutableMapping):
         :type default: any
         """
         value = self.get(name, default or [])
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = value.split(',')
         return list(value)
 
@@ -194,7 +193,7 @@ class BaseSettings(MutableMapping):
         :type default: any
         """
         value = self.get(name, default or {})
-        if isinstance(value, six.string_types):
+        if isinstance(value, str):
             value = json.loads(value)
         return dict(value)
 
@@ -284,7 +283,7 @@ class BaseSettings(MutableMapping):
         :type priority: string or int
         """
         self._assert_mutability()
-        if isinstance(module, six.string_types):
+        if isinstance(module, str):
             module = import_module(module)
         for key in dir(module):
             if key.isupper():
@@ -313,14 +312,14 @@ class BaseSettings(MutableMapping):
         :type priority: string or int
         """
         self._assert_mutability()
-        if isinstance(values, six.string_types):
+        if isinstance(values, str):
             values = json.loads(values)
         if values is not None:
             if isinstance(values, BaseSettings):
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, values.getpriority(name))
             else:
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, priority)
 
     def delete(self, name, priority='project'):
@@ -377,7 +376,7 @@ class BaseSettings(MutableMapping):
 
     def _to_dict(self):
         return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
-                for k, v in six.iteritems(self)}
+                for k, v in self.items()}
 
     def copy_to_dict(self):
         """
@@ -445,7 +444,7 @@ class Settings(BaseSettings):
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
-        for name, val in six.iteritems(self):
+        for name, val in self.items():
             if isinstance(val, dict):
                 self.set(name, BaseSettings(val, 'default'), 'default')
         self.update(values, priority)

scrapy/shell.py

@@ -3,8 +3,6 @@
 See documentation in docs/topics/shell.rst
 
 """
-from __future__ import print_function
-
 import os
 import signal
 import warnings

scrapy/signalmanager.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 from pydispatch import dispatcher
 from scrapy.utils import signal as _signal
 

scrapy/spiderloader.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
 from collections import defaultdict
 import traceback
 import warnings

scrapy/spidermiddlewares/referer.py

@@ -2,8 +2,8 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import urlparse
 import warnings
+from urllib.parse import urlparse
 
 from w3lib.url import safe_url_string
 

scrapy/spiders/__init__.py

@@ -57,6 +57,11 @@ class Spider(object_ref):
 
     def start_requests(self):
         cls = self.__class__
+        if not self.start_urls and hasattr(self, 'start_url'):
+            raise AttributeError(
+                "Crawling could not start: 'start_urls' not found "
+                "or empty (but found 'start_url' attribute instead, "
+                "did you miss an 's'?)")
         if method_is_overridden(cls, Spider, 'make_requests_from_url'):
             warnings.warn(
                 "Spider.make_requests_from_url method is deprecated; it "

scrapy/spiders/crawl.py

@@ -8,8 +8,6 @@ See documentation in docs/topics/spiders.rst
 import copy
 import warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
@@ -25,7 +23,7 @@ def _identity(request, response):
 def _get_method(method, spider):
     if callable(method):
         return method
-    elif isinstance(method, six.string_types):
+    elif isinstance(method, str):
         return getattr(spider, method, None)
 
 

scrapy/spiders/sitemap.py

@@ -1,6 +1,5 @@
 import re
 import logging
-import six
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, XmlResponse
@@ -22,7 +21,7 @@ class SitemapSpider(Spider):
         super(SitemapSpider, self).__init__(*a, **kw)
         self._cbs = []
         for r, c in self.sitemap_rules:
-            if isinstance(c, six.string_types):
+            if isinstance(c, str):
                 c = getattr(self, c)
             self._cbs.append((regex(r), c))
         self._follow = [regex(x) for x in self.sitemap_follow]
@@ -86,7 +85,7 @@ class SitemapSpider(Spider):
 
 
 def regex(x):
-    if isinstance(x, six.string_types):
+    if isinstance(x, str):
         return re.compile(x)
     return x
 

scrapy/squeues.py

@@ -3,7 +3,7 @@ Scheduler queues
 """
 
 import marshal
-from six.moves import cPickle as pickle
+import pickle
 
 from queuelib import queue
 

scrapy/utils/benchserver.py

@@ -1,5 +1,6 @@
 import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
 from twisted.web.server import Site
 from twisted.web.resource import Resource
 from twisted.internet import reactor

scrapy/utils/boto.py

@@ -1,7 +1,5 @@
 """Boto/botocore helpers"""
 
-from __future__ import absolute_import
-
 from scrapy.exceptions import NotConfigured
 
 

scrapy/utils/conf.py

@@ -1,11 +1,9 @@
-from configparser import ConfigParser
 import os
 import sys
 import numbers
+from configparser import ConfigParser
 from operator import itemgetter
 
-import six
-
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
     def _map_keys(compdict):
         if isinstance(compdict, BaseSettings):
             compbs = BaseSettings()
-            for k, v in six.iteritems(compdict):
+            for k, v in compdict.items():
                 prio = compdict.getpriority(k)
                 if compbs.getpriority(convert(k)) == prio:
                     raise ValueError('Some paths in {!r} convert to the same '
@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
             return compbs
         else:
             _check_components(compdict)
-            return {convert(k): v for k, v in six.iteritems(compdict)}
+            return {convert(k): v for k, v in compdict.items()}
 
     def _validate_values(compdict):
         """Fail if a value in the components dict is not a real number or None."""
-        for name, value in six.iteritems(compdict):
+        for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
                 raise ValueError('Invalid value {} for component {}, please provide ' \
                                  'a real number or None instead'.format(value, name))
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
 
     _validate_values(compdict)
     compdict = without_none_values(_map_keys(compdict))
-    return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+    return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
 
 
 def arglist_to_dict(arglist):

scrapy/utils/curl.py

@@ -1,10 +1,9 @@
 import argparse
 import warnings
 from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
 
-from six.moves.http_cookies import SimpleCookie
-from six.moves.urllib.parse import urlparse
-from six import iteritems
 from w3lib.http import basic_auth_header
 
 
@@ -76,7 +75,7 @@ def curl_to_request_kwargs(curl_command, ignore_unknown_options=True):
         name = name.strip()
         val = val.strip()
         if name.title() == 'Cookie':
-            for name, morsel in iteritems(SimpleCookie(val)):
+            for name, morsel in SimpleCookie(val).items():
                 cookies[name] = morsel.value
         else:
             headers.append((name, val))

scrapy/utils/datatypes.py

@@ -5,12 +5,10 @@ Python Standard Library.
 This module must not depend on any module outside the Standard Library.
 """
 
-import copy
 import collections
-from collections.abc import Mapping
+import copy
 import warnings
-
-import six
+from collections.abc import Mapping
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -151,7 +149,7 @@ class MultiValueDict(dict):
                         self.setlistdefault(key, []).append(value)
                 except TypeError:
                     raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in six.iteritems(kwargs):
+        for key, value in kwargs.items():
             self.setlistdefault(key, []).append(value)
 
 
@@ -238,65 +236,6 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class MergeDict(object):
-    """
-    A simple class for creating new "virtual" dictionaries that actually look
-    up values in more than one dictionary, passed in the ``__init__`` method.
-
-    If a key appears in more than one of the given dictionaries, only the
-    first occurrence will be used.
-    """
-    def __init__(self, *dicts):
-        warnings.warn(
-            "scrapy.utils.datatypes.MergeDict is deprecated in favor "
-            "of collections.ChainMap (introduced in Python 3.3)",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        self.dicts = dicts
-
-    def __getitem__(self, key):
-        for dict_ in self.dicts:
-            try:
-                return dict_[key]
-            except KeyError:
-                pass
-        raise KeyError
-
-    def __copy__(self):
-        return self.__class__(*self.dicts)
-
-    def get(self, key, default=None):
-        try:
-            return self[key]
-        except KeyError:
-            return default
-
-    def getlist(self, key):
-        for dict_ in self.dicts:
-            if key in dict_.keys():
-                return dict_.getlist(key)
-        return []
-
-    def items(self):
-        item_list = []
-        for dict_ in self.dicts:
-            item_list.extend(dict_.items())
-        return item_list
-
-    def has_key(self, key):
-        for dict_ in self.dicts:
-            if key in dict_:
-                return True
-        return False
-
-    __contains__ = has_key
-
-    def copy(self):
-        """Returns a copy of this object."""
-        return self.__copy__()
-
-
 class LocalCache(collections.OrderedDict):
     """Dictionary with a finite number of keys.
 

scrapy/utils/display.py

@@ -2,7 +2,6 @@
 pprint and pformat wrappers with colorization support
 """
 
-from __future__ import print_function
 import sys
 from pprint import pformat as pformat_
 

scrapy/utils/httpobj.py

@@ -1,8 +1,7 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
 import weakref
-
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 
 _urlparse_cache = weakref.WeakKeyDictionary()

scrapy/utils/iterators.py

@@ -1,13 +1,13 @@
-import re
 import csv
-from io import StringIO
 import logging
-import six
+import re
+from io import StringIO
 
 from scrapy.http import TextResponse, Response
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -60,7 +60,7 @@ class _StreamReader(object):
             self._text, self.encoding = obj.body, obj.encoding
         else:
             self._text, self.encoding = obj, 'utf-8'
-        self._is_unicode = isinstance(self._text, six.text_type)
+        self._is_unicode = isinstance(self._text, str)
 
     def read(self, n=65535):
         self.read = self._read_unicode if self._is_unicode else self._read_string
@@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
 
 
 def _body_or_str(obj, unicode=True):
-    expected_types = (Response, six.text_type, six.binary_type)
+    expected_types = (Response, str, bytes)
     assert isinstance(obj, expected_types), \
         "obj must be %s, not %s" % (
             " or ".join(t.__name__ for t in expected_types),
@@ -137,7 +137,7 @@ def _body_or_str(obj, unicode=True):
             return obj.text
         else:
             return obj.body.decode('utf-8')
-    elif isinstance(obj, six.text_type):
+    elif isinstance(obj, str):
         return obj if unicode else obj.encode('utf-8')
     else:
         return obj.decode('utf-8') if unicode else obj

scrapy/utils/misc.py

@@ -6,14 +6,13 @@ from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
 
-import six
 from w3lib.html import replace_entities
 
 from scrapy.utils.python import flatten, to_unicode
 from scrapy.item import BaseItem
 
 
-_ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes
+_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
 
 
 def arg_to_iter(arg):
@@ -83,7 +82,7 @@ def extract_regex(regex, text, encoding='utf-8'):
     * if the regex doesn't contain any group the entire regex matching is returned
     """
 
-    if isinstance(regex, six.string_types):
+    if isinstance(regex, str):
         regex = re.compile(regex, re.UNICODE)
 
     try:
@@ -92,7 +91,7 @@ def extract_regex(regex, text, encoding='utf-8'):
         strings = regex.findall(text)    # full regex or numbered groups
     strings = flatten(strings)
 
-    if isinstance(text, six.text_type):
+    if isinstance(text, str):
         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
     else:
         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])

scrapy/utils/ossignal.py

@@ -1,5 +1,3 @@
-
-from __future__ import absolute_import
 import signal
 
 from twisted.internet import reactor

scrapy/utils/project.py

@@ -1,5 +1,5 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 import warnings
 
 from importlib import import_module
@@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'

scrapy/utils/python.py

@@ -7,7 +7,6 @@ import re
 import inspect
 import weakref
 import errno
-import six
 from functools import partial, wraps
 from itertools import chain
 import sys
@@ -65,10 +64,10 @@ def is_listlike(x):
     True
     >>> is_listlike((x for x in range(3)))
     True
-    >>> is_listlike(six.moves.xrange(5))
+    >>> is_listlike(range(5))
     True
     """
-    return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))
+    return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
 def unique(list_, key=lambda x: x):
@@ -87,9 +86,9 @@ def unique(list_, key=lambda x: x):
 def to_unicode(text, encoding=None, errors='strict'):
     """Return the unicode representation of a bytes object ``text``. If
     ``text`` is already an unicode object, return it as-is."""
-    if isinstance(text, six.text_type):
+    if isinstance(text, str):
         return text
-    if not isinstance(text, (bytes, six.text_type)):
+    if not isinstance(text, (bytes, str)):
         raise TypeError('to_unicode must receive a bytes or str '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
@@ -102,7 +101,7 @@ def to_bytes(text, encoding=None, errors='strict'):
     is already a bytes object, return it as-is."""
     if isinstance(text, bytes):
         return text
-    if not isinstance(text, six.string_types):
+    if not isinstance(text, str):
         raise TypeError('to_bytes must receive a str or bytes '
                         'object, got %s' % type(text).__name__)
     if encoding is None:
@@ -138,7 +137,7 @@ def re_rsearch(pattern, text, chunk_size=1024):
             yield (text[offset:], offset)
         yield (text, 0)
 
-    if isinstance(pattern, six.string_types):
+    if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
     for chunk, offset in _chunk_iter():
@@ -162,7 +161,7 @@ def memoizemethod_noargs(method):
     return new_method
 
 
-_BINARYCHARS = {six.b(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
+_BINARYCHARS = {to_bytes(chr(i)) for i in range(32)} - {b"\0", b"\t", b"\n", b"\r"}
 _BINARYCHARS |= {ord(ch) for ch in _BINARYCHARS}
 
 
@@ -301,10 +300,10 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     dict or a list of tuples, like any dict ``__init__`` method supports.
     """
     d = {}
-    for k, v in six.iteritems(dict(dct_or_tuples)):
-        k = k.encode(encoding) if isinstance(k, six.text_type) else k
+    for k, v in dict(dct_or_tuples).items():
+        k = k.encode(encoding) if isinstance(k, str) else k
         if not keys_only:
-            v = v.encode(encoding) if isinstance(v, six.text_type) else v
+            v = v.encode(encoding) if isinstance(v, str) else v
         d[k] = v
     return d
 
@@ -346,7 +345,7 @@ def without_none_values(iterable):
     value ``None`` have been removed.
     """
     try:
-        return {k: v for k, v in six.iteritems(iterable) if v is not None}
+        return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
         return type(iterable)((v for v in iterable if v is not None))
 

scrapy/utils/reqser.py

@@ -1,8 +1,6 @@
 """
 Helper functions for serializing (and deserializing) requests.
 """
-import six
-
 from scrapy.http import Request
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
@@ -87,12 +85,12 @@ def _mangle_private_name(obj, func, name):
 def _find_method(obj, func):
     if obj:
         try:
-            func_self = six.get_method_self(func)
+            func_self = func.__self__
         except AttributeError:  # func has no __self__
             pass
         else:
             if func_self is obj:
-                name = six.get_method_function(func).__name__
+                name = func.__func__.__name__
                 if _is_private_method(name):
                     return _mangle_private_name(obj, func, name)
                 return name

scrapy/utils/request.py

@@ -3,16 +3,15 @@ This module provides some useful functions for working with
 scrapy.http.Request objects
 """
 
-from __future__ import print_function
 import hashlib
 import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_unicode
-
 from w3lib.url import canonicalize_url
+
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 _fingerprint_cache = weakref.WeakKeyDictionary()

scrapy/utils/sitemap.py

@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
+from urllib.parse import urljoin
+
 import lxml.etree
-from six.moves.urllib.parse import urljoin
 
 
 class Sitemap(object):

scrapy/utils/spider.py

@@ -1,11 +1,10 @@
 import logging
 import inspect
 
-import six
-
 from scrapy.spiders import Spider
 from scrapy.utils.misc import  arg_to_iter
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
     # singleton in scrapy.spider.spiders
     from scrapy.spiders import Spider
 
-    for obj in six.itervalues(vars(module)):
+    for obj in vars(module).values():
         if inspect.isclass(obj) and \
            issubclass(obj, Spider) and \
            obj.__module__ == module.__name__ and \

scrapy/utils/test.py

@@ -2,7 +2,6 @@
 This module contains some assorted functions used in tests
 """
 
-from __future__ import absolute_import
 import os
 
 from importlib import import_module

scrapy/utils/testproc.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import
 import sys
 import os
 

scrapy/utils/testsite.py

@@ -1,5 +1,4 @@
-from __future__ import print_function
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from twisted.internet import reactor
 from twisted.web import server, resource, static, util

scrapy/utils/trackref.py

@@ -9,12 +9,10 @@ and no performance penalty at all when disabled (as object_ref becomes just an
 alias to object in that case).
 """
 
-from __future__ import print_function
 import weakref
 from time import time
 from operator import itemgetter
 from collections import defaultdict
-import six
 
 
 NoneType = type(None)
@@ -37,13 +35,13 @@ def format_live_refs(ignore=NoneType):
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
-    for cls, wdict in sorted(six.iteritems(live_refs),
+    for cls, wdict in sorted(live_refs.items(),
                              key=lambda x: x[0].__name__):
         if not wdict:
             continue
         if issubclass(cls, ignore):
             continue
-        oldest = min(six.itervalues(wdict))
+        oldest = min(wdict.values())
         s += "%-30s %6d   oldest: %ds ago\n" % (
             cls.__name__, len(wdict), now - oldest
         )
@@ -57,15 +55,15 @@ def print_live_refs(*a, **kw):
 
 def get_oldest(class_name):
     """Get the oldest object for a specific class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             if not wdict:
                 break
-            return min(six.iteritems(wdict), key=itemgetter(1))[0]
+            return min(wdict.items(), key=itemgetter(1))[0]
 
 
 def iter_all(class_name):
     """Iterate over all objects of the same class by its class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
-            return six.iterkeys(wdict)
+            return wdict.keys()

scrapy/utils/url.py

@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code

setup.py

@@ -72,7 +72,6 @@ setup(
         'pyOpenSSL>=16.2.0',
         'queuelib>=1.4.2',
         'service_identity>=16.0.0',
-        'six>=1.10.0',
         'w3lib>=1.17.0',
         'zope.interface>=4.1.3',
         'protego>=0.1.15',

tests/CrawlerProcess/simple.py

@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()

tests/mockserver.py

@@ -3,9 +3,9 @@ import os
 import random
 import sys
 from subprocess import Popen, PIPE
+from urllib.parse import urlencode
 
 from OpenSSL import SSL
-from six.moves.urllib.parse import urlencode
 
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource

tests/requirements-py3.txt

@@ -1,7 +1,7 @@
 # Tests requirements
 jmespath
 mitmproxy; python_version >= '3.6'
-mitmproxy==3.0.4; python_version < '3.6'
+mitmproxy<4.0.0; python_version < '3.6'
 pytest
 pytest-cov
 pytest-twisted >= 1.11

tests/spiders.py

@@ -3,7 +3,7 @@ Some spiders used for testing and benchmarking
 """
 
 import time
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 from scrapy.spiders import Spider
 from scrapy.http import Request

tests/test_cmdline/__init__.py

@@ -1,13 +1,12 @@
-from io import StringIO
 import json
 import os
 import pstats
 import shutil
-import six
-from subprocess import Popen, PIPE
 import sys
 import tempfile
 import unittest
+from io import StringIO
+from subprocess import Popen, PIPE
 
 from scrapy.utils.test import get_testenv
 
@@ -65,5 +64,5 @@ class CmdlineTest(unittest.TestCase):
         for char in ("'", "<", ">", 'u"'):
             settingsstr = settingsstr.replace(char, '"')
         settingsdict = json.loads(settingsstr)
-        six.assertCountEqual(self, settingsdict.keys(), EXTENSIONS.keys())
+        self.assertCountEqual(settingsdict.keys(), EXTENSIONS.keys())
         self.assertEqual(200, settingsdict[EXT_PATH])

tests/test_contracts.py

@@ -1,6 +1,5 @@
 from unittest import TextTestResult
 
-from six import get_unbound_function
 from twisted.internet import defer
 from twisted.python import failure
 from twisted.trial import unittest
@@ -395,8 +394,8 @@ class ContractsManagerTest(unittest.TestCase):
         with MockServer() as mockserver:
             contract_doc = '@url {}'.format(mockserver.url('/status?n=200'))
 
-            get_unbound_function(TestSameUrlSpider.parse_first).__doc__ = contract_doc
-            get_unbound_function(TestSameUrlSpider.parse_second).__doc__ = contract_doc
+            TestSameUrlSpider.parse_first.__doc__ = contract_doc
+            TestSameUrlSpider.parse_second.__doc__ = contract_doc
 
             crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
             yield crawler.crawl()

tests/test_crawl.py

@@ -140,7 +140,7 @@ class CrawlTestCase(TestCase):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from six.moves.urllib.parse import urlencode
+        from urllib.parse import urlencode
         query = urlencode({'raw': '''\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1

tests/test_crawler.py

@@ -1,4 +1,7 @@
 import logging
+import os
+import subprocess
+import sys
 import warnings
 
 from pytest import raises, mark
@@ -16,6 +19,7 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
 
 
 class BaseCrawlerTest(unittest.TestCase):
@@ -281,3 +285,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         with LogCapture() as log:
             yield runner.crawl(AsyncioSpider)
             log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+    def run_script(self, script_name):
+        script_path = os.path.join(self.script_dir, script_name)
+        args = (sys.executable, script_path)
+        p = subprocess.Popen(args, env=get_testenv(),
+                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = p.communicate()
+        return stderr.decode('utf-8')
+
+    def test_simple(self):
+        log = self.run_script('simple.py')
+        self.assertIn('Spider closed (finished)', log)

tests/test_downloadermiddleware_robotstxt.py

@@ -1,6 +1,4 @@
 # -*- coding: utf-8 -*-
-from __future__ import absolute_import
-
 from unittest import mock
 
 from twisted.internet import reactor, error

tests/test_engine.py

@@ -10,9 +10,10 @@ module with the ``runserver`` argument::
     python test_engine.py runserver
 """
 
-from __future__ import print_function
-import sys, os, re
-from six.moves.urllib.parse import urlparse
+import os
+import re
+import sys
+from urllib.parse import urlparse
 
 from twisted.internet import reactor, defer
 from twisted.web import server, static, util

tests/test_exporters.py

@@ -1,15 +1,13 @@
-from __future__ import absolute_import
 import re
 import json
 import marshal
+import pickle
 import tempfile
 import unittest
 from io import BytesIO
 from datetime import datetime
-from six.moves import cPickle as pickle
 
 import lxml.etree
-import six
 
 from scrapy.item import Item, Field
 from scrapy.utils.python import to_unicode
@@ -80,7 +78,7 @@ class BaseItemExporterTest(unittest.TestCase):
 
         ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
         _, name = list(ie._get_serialized_fields(self.i))[0]
-        assert isinstance(name, six.text_type)
+        assert isinstance(name, str)
         self.assertEqual(name, u'John\xa3')
 
     def test_field_custom_serializer(self):

tests/test_feedexport.py

@@ -1,15 +1,15 @@
-from __future__ import absolute_import
 import os
 import csv
 import json
 import warnings
-from io import BytesIO
 import tempfile
 import shutil
 import string
+from io import BytesIO
+from pathlib import Path
 from unittest import mock
-from six.moves.urllib.parse import urljoin, urlparse, quote
-from six.moves.urllib.request import pathname2url
+from urllib.parse import urljoin, urlparse, quote
+from urllib.request import pathname2url
 
 from zope.interface.verify import verifyObject
 from twisted.trial import unittest
@@ -28,8 +28,6 @@ from scrapy.extensions.feedexport import (
 from scrapy.utils.test import assert_aws_environ, get_s3_content_and_delete, get_crawler
 from scrapy.utils.python import to_unicode
 
-from pathlib import Path
-
 
 class FileFeedStorageTest(unittest.TestCase):