Remove six.moves occurrences

scrapy/_monkeypatches.py

@@ -1,4 +1,4 @@
-from six.moves import copyreg
+import copyreg
 
 
 # Undo what Twisted's perspective broker adds to pickle register

scrapy/commands/bench.py

@@ -1,8 +1,7 @@
 import sys
 import time
 import subprocess
-
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 
 import scrapy
 from scrapy.commands import ScrapyCommand

scrapy/core/downloader/handlers/ftp.py

@@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys:
 
 import re
 from io import BytesIO
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from twisted.internet import reactor
 from twisted.protocols.ftp import FTPClient, CommandFailed

scrapy/core/downloader/handlers/http11.py

@@ -2,10 +2,10 @@
 
 import re
 import logging
+import warnings
 from io import BytesIO
 from time import time
-import warnings
-from six.moves.urllib.parse import urldefrag
+from urllib.parse import urldefrag
 
 from zope.interface import implementer
 from twisted.internet import defer, reactor, protocol

scrapy/core/downloader/handlers/s3.py

@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import unquote
+from urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/core/downloader/webclient.py

@@ -1,5 +1,5 @@
 from time import time
-from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+from urllib.parse import urlparse, urlunparse, urldefrag
 
 from twisted.web.client import HTTPClientFactory
 from twisted.web.http import HTTPClient

scrapy/downloadermiddlewares/httpproxy.py

@@ -1,7 +1,6 @@
 import base64
-from six.moves.urllib.parse import unquote, urlunparse
-from six.moves.urllib.request import getproxies, proxy_bypass
-from urllib.request import _parse_proxy
+from urllib.parse import unquote, urlunparse
+from urllib.request import getproxies, proxy_bypass, _parse_proxy
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/downloadermiddlewares/redirect.py

@@ -1,5 +1,5 @@
 import logging
-from six.moves.urllib.parse import urljoin, urlparse
+from urllib.parse import urljoin, urlparse
 
 from w3lib.url import safe_url_string
 
@@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse
 from scrapy.utils.response import get_meta_refresh
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 
+
 logger = logging.getLogger(__name__)
 
 

scrapy/exporters.py

@@ -7,15 +7,16 @@ import io
 import sys
 import pprint
 import marshal
-import six
-from six.moves import cPickle as pickle
+import warnings
+from import pickle
 from xml.sax.saxutils import XMLGenerator
 
+import six
+
 from scrapy.utils.serialize import ScrapyJSONEncoder
 from scrapy.utils.python import to_bytes, to_unicode, is_listlike
 from scrapy.item import BaseItem
 from scrapy.exceptions import ScrapyDeprecationWarning
-import warnings
 
 
 __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',

scrapy/extensions/feedexport.py

@@ -10,7 +10,7 @@ import logging
 import posixpath
 from tempfile import NamedTemporaryFile
 from datetime import datetime
-from six.moves.urllib.parse import urlparse, unquote
+from urllib.parse import urlparse, unquote
 from ftplib import FTP
 
 from zope.interface import Interface, implementer

scrapy/extensions/httpcache.py

@@ -1,13 +1,13 @@
 import gzip
 import logging
 import os
+import pickle
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from time import time
 from warnings import warn
 from weakref import WeakKeyDictionary
 
-from six.moves import cPickle as pickle
 from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
 
 from scrapy.exceptions import ScrapyDeprecationWarning

scrapy/extensions/spiderstate.py

@@ -1,10 +1,11 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
+
 class SpiderState(object):
     """Store and load spider state during a scraping job"""
 

scrapy/http/cookies.py

@@ -1,7 +1,6 @@
 import time
-from six.moves.http_cookiejar import (
-    CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
-)
+from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE
+
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 

scrapy/http/request/form.py

@@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin, urlencode
+from urllib.parse import urljoin, urlencode
 
 import lxml.html
+import six
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 

scrapy/http/request/rpc.py

@@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves import xmlrpc_client as xmlrpclib
+import xmlrpc.client as xmlrpclib
 
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args

scrapy/http/response/__init__.py

@@ -4,7 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from scrapy.http.request import Request
 from scrapy.http.headers import Headers

scrapy/http/response/text.py

@@ -5,10 +5,10 @@ discovering (through HTTP headers) to base Response class.
 See documentation in docs/topics/request-response.rst
 """
 
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 import parsel
+import six
 from w3lib.encoding import html_to_unicode, resolve_encoding, \
     html_body_declared_encoding, http_content_type_encoding
 from w3lib.html import strip_html5_whitespace

scrapy/linkextractors/__init__.py

@@ -6,8 +6,8 @@ This package contains a collection of Link Extractors.
 For more info see docs/topics/link-extractors.rst
 """
 import re
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 from parsel.csstranslator import HTMLTranslator
 from w3lib.url import canonicalize_url
 

scrapy/linkextractors/htmlparser.py

@@ -2,10 +2,10 @@
 HTMLParser-based link extractor
 """
 import warnings
-import six
-from six.moves.html_parser import HTMLParser
-from six.moves.urllib.parse import urljoin
+from html.parser import HTMLParser
+from urllib.parse import urljoin
 
+import six
 from w3lib.url import safe_url_string
 from w3lib.html import strip_html5_whitespace
 

scrapy/linkextractors/lxmlhtml.py

@@ -1,9 +1,9 @@
 """
 Link extractor based on lxml.html
 """
-import six
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
+import six
 import lxml.etree as etree
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url

scrapy/linkextractors/regex.py

@@ -1,11 +1,12 @@
 import re
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url
 
 from scrapy.link import Link
 from .sgml import SgmlLinkExtractor
 
+
 linkre = re.compile(
         "<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
         re.DOTALL | re.IGNORECASE)

scrapy/linkextractors/sgml.py

@@ -1,11 +1,11 @@
 """
 SGMLParser-based Link extractors
 """
-import six
-from six.moves.urllib.parse import urljoin
 import warnings
+from urllib.parse import urljoin
 from sgmllib import SGMLParser
 
+import six
 from w3lib.url import safe_url_string, canonicalize_url
 from w3lib.html import strip_html5_whitespace
 

scrapy/mail.py

@@ -3,21 +3,21 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
-from io import BytesIO
 import logging
-
-from email.utils import COMMASPACE, formatdate
-from six.moves.email_mime_multipart import MIMEMultipart
-from six.moves.email_mime_text import MIMEText
-from six.moves.email_mime_base import MIMEBase
-from email.mime.nonmultipart import MIMENonMultipart
 from email import encoders as Encoders
+from email.mime.base import MIMEBase
+from email.mime.multipart import MIMEMultipart
+from email.mime.nonmultipart import MIMENonMultipart
+from email.mime.text import MIMEText
+from email.utils import COMMASPACE, formatdate
+from io import BytesIO
 
 from twisted.internet import defer, reactor, ssl
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
+
 logger = logging.getLogger(__name__)
 
 

scrapy/pipelines/files.py

@@ -5,17 +5,17 @@ See documentation in topics/media-pipeline.rst
 """
 import functools
 import hashlib
-from io import BytesIO
+import logging
 import mimetypes
 import os
 import os.path
 import time
-import logging
-from email.utils import parsedate_tz, mktime_tz
-from six.moves.urllib.parse import urlparse
 from collections import defaultdict
-import six
+from email.utils import parsedate_tz, mktime_tz
+from io import BytesIO
+from urllib.parse import urlparse
 
+import six
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline

scrapy/robotstxt.py

@@ -53,7 +53,7 @@ class RobotParser(with_metaclass(ABCMeta)):
 
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body, spider):
-        from six.moves.urllib_robotparser import RobotFileParser
+        from urllib.robotparser import RobotFileParser
         self.spider = spider
         robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp = RobotFileParser()

scrapy/spidermiddlewares/referer.py

@@ -2,8 +2,8 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
-from six.moves.urllib.parse import urlparse
 import warnings
+from urllib.parse import urlparse
 
 from w3lib.url import safe_url_string
 

scrapy/squeues.py

@@ -3,7 +3,7 @@ Scheduler queues
 """
 
 import marshal
-from six.moves import cPickle as pickle
+import pickle
 
 from queuelib import queue
 

scrapy/utils/benchserver.py

@@ -1,5 +1,6 @@
 import random
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
+
 from twisted.web.server import Site
 from twisted.web.resource import Resource
 from twisted.internet import reactor

scrapy/utils/curl.py

@@ -1,9 +1,9 @@
 import argparse
 import warnings
 from shlex import split
+from http.cookies import SimpleCookie
+from urllib.parse import urlparse
 
-from six.moves.http_cookies import SimpleCookie
-from six.moves.urllib.parse import urlparse
 from six import string_types, iteritems
 from w3lib.http import basic_auth_header
 

scrapy/utils/httpobj.py

@@ -1,8 +1,8 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
 import weakref
+from urllib.parse import urlparse
 
-from six.moves.urllib.parse import urlparse
 
 _urlparse_cache = weakref.WeakKeyDictionary()
 def urlparse_cached(request_or_response):

scrapy/utils/project.py

@@ -1,5 +1,5 @@
 import os
-from six.moves import cPickle as pickle
+import pickle
 import warnings
 
 from importlib import import_module
@@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists
 
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import Settings
-from scrapy.exceptions import NotConfigured
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+
 
 ENVVAR = 'SCRAPY_SETTINGS_MODULE'
 DATADIR_CFG_SECTION = 'datadir'

scrapy/utils/python.py

@@ -65,7 +65,7 @@ def is_listlike(x):
     True
     >>> is_listlike((x for x in range(3)))
     True
-    >>> is_listlike(six.moves.xrange(5))
+    >>> is_listlike(range(5))
     True
     """
     return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))

scrapy/utils/request.py

@@ -5,13 +5,13 @@ scrapy.http.Request objects
 
 import hashlib
 import weakref
-from six.moves.urllib.parse import urlunparse
+from urllib.parse import urlunparse
 
 from w3lib.http import basic_auth_header
-from scrapy.utils.python import to_bytes, to_unicode
-
 from w3lib.url import canonicalize_url
+
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.python import to_bytes, to_unicode
 
 
 _fingerprint_cache = weakref.WeakKeyDictionary()

scrapy/utils/sitemap.py

@@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
+from urllib.parse import urljoin
+
 import lxml.etree
-from six.moves.urllib.parse import urljoin
 
 
 class Sitemap(object):

scrapy/utils/testsite.py

@@ -1,4 +1,4 @@
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from twisted.internet import reactor
 from twisted.web import server, resource, static, util

scrapy/utils/url.py

@@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead.
 """
 import posixpath
 import re
-from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse)
+from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 # scrapy.utils.url was moved to w3lib.url and import * ensures this
 # move doesn't break old code