diff --git a/scrapy/_monkeypatches.py b/scrapy/_monkeypatches.py index 1f8067b35..f74f89bda 100644 --- a/scrapy/_monkeypatches.py +++ b/scrapy/_monkeypatches.py @@ -1,4 +1,4 @@ -from six.moves import copyreg +import copyreg # Undo what Twisted's perspective broker adds to pickle register diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 90c8d56a2..7bbe362e7 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -1,8 +1,7 @@ import sys import time import subprocess - -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode import scrapy from scrapy.commands import ScrapyCommand diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 806a537d4..2116a5a44 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -30,7 +30,7 @@ In case of status 200 request, response.headers will come with two keys: import re from io import BytesIO -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from twisted.internet import reactor from twisted.protocols.ftp import FTPClient, CommandFailed diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7d917cb74..63dedc19b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -2,10 +2,10 @@ import re import logging +import warnings from io import BytesIO from time import time -import warnings -from six.moves.urllib.parse import urldefrag +from urllib.parse import urldefrag from zope.interface import implementer from twisted.internet import defer, reactor, protocol diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d8bbdd326..85b733228 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import unquote +from urllib.parse import unquote from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 3a5890ed0..9699da109 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,5 @@ from time import time -from six.moves.urllib.parse import urlparse, urlunparse, urldefrag +from urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 2212d9688..814ce78fe 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -1,7 +1,6 @@ import base64 -from six.moves.urllib.parse import unquote, urlunparse -from six.moves.urllib.request import getproxies, proxy_bypass -from urllib.request import _parse_proxy +from urllib.parse import unquote, urlunparse +from urllib.request import getproxies, proxy_bypass, _parse_proxy from scrapy.exceptions import NotConfigured from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index b73f864dd..77cb5aa94 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -1,5 +1,5 @@ import logging -from six.moves.urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse from w3lib.url import safe_url_string @@ -7,6 +7,7 @@ from scrapy.http import HtmlResponse from scrapy.utils.response import get_meta_refresh from scrapy.exceptions import IgnoreRequest, NotConfigured + logger = logging.getLogger(__name__) diff --git a/scrapy/exporters.py b/scrapy/exporters.py index f276c28e8..d9531e67c 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -7,15 +7,16 @@ import io import sys import pprint import marshal -import six -from six.moves import cPickle as pickle +import warnings +from import pickle from xml.sax.saxutils import XMLGenerator +import six + from scrapy.utils.serialize import ScrapyJSONEncoder from scrapy.utils.python import to_bytes, to_unicode, is_listlike from scrapy.item import BaseItem from scrapy.exceptions import ScrapyDeprecationWarning -import warnings __all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter', diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 07ffd3476..0b854e6f3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -10,7 +10,7 @@ import logging import posixpath from tempfile import NamedTemporaryFile from datetime import datetime -from six.moves.urllib.parse import urlparse, unquote +from urllib.parse import urlparse, unquote from ftplib import FTP from zope.interface import Interface, implementer diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index b1ed0b9f8..b98ec218f 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,13 +1,13 @@ import gzip import logging import os +import pickle from email.utils import mktime_tz, parsedate_tz from importlib import import_module from time import time from warnings import warn from weakref import WeakKeyDictionary -from six.moves import cPickle as pickle from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.exceptions import ScrapyDeprecationWarning diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py index 2220cbd8f..2c8e46914 100644 --- a/scrapy/extensions/spiderstate.py +++ b/scrapy/extensions/spiderstate.py @@ -1,10 +1,11 @@ import os -from six.moves import cPickle as pickle +import pickle from scrapy import signals from scrapy.exceptions import NotConfigured from scrapy.utils.job import job_dir + class SpiderState(object): """Store and load spider state during a scraping job""" diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 4532c3ab7..c39de0b52 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -1,7 +1,6 @@ import time -from six.moves.http_cookiejar import ( - CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE -) +from http.cookiejar import CookieJar as _CookieJar, DefaultCookiePolicy, IPV4_RE + from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_unicode diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index b6feede07..d2bcf77b7 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin, urlencode +from urllib.parse import urljoin, urlencode import lxml.html +import six from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index bd09f7534..811d3ad6b 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,7 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from six.moves import xmlrpc_client as xmlrpclib +import xmlrpc.client as xmlrpclib from scrapy.http.request import Request from scrapy.utils.python import get_func_args diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a81404afb..64e9c6c20 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -4,7 +4,7 @@ responses in Scrapy. See documentation in docs/topics/request-response.rst """ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from scrapy.http.request import Request from scrapy.http.headers import Headers diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 37f450e54..69bcba577 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -5,10 +5,10 @@ discovering (through HTTP headers) to base Response class. See documentation in docs/topics/request-response.rst """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin import parsel +import six from w3lib.encoding import html_to_unicode, resolve_encoding, \ html_body_declared_encoding, http_content_type_encoding from w3lib.html import strip_html5_whitespace diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index ebf3cd7d8..594c7264d 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -6,8 +6,8 @@ This package contains a collection of Link Extractors. For more info see docs/topics/link-extractors.rst """ import re +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse from parsel.csstranslator import HTMLTranslator from w3lib.url import canonicalize_url diff --git a/scrapy/linkextractors/htmlparser.py b/scrapy/linkextractors/htmlparser.py index 27978a8a1..623732cee 100644 --- a/scrapy/linkextractors/htmlparser.py +++ b/scrapy/linkextractors/htmlparser.py @@ -2,10 +2,10 @@ HTMLParser-based link extractor """ import warnings -import six -from six.moves.html_parser import HTMLParser -from six.moves.urllib.parse import urljoin +from html.parser import HTMLParser +from urllib.parse import urljoin +import six from w3lib.url import safe_url_string from w3lib.html import strip_html5_whitespace diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 890c019c8..5b7e709de 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -1,9 +1,9 @@ """ Link extractor based on lxml.html """ -import six -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin +import six import lxml.etree as etree from w3lib.html import strip_html5_whitespace from w3lib.url import canonicalize_url diff --git a/scrapy/linkextractors/regex.py b/scrapy/linkextractors/regex.py index e689b4727..f96db256b 100644 --- a/scrapy/linkextractors/regex.py +++ b/scrapy/linkextractors/regex.py @@ -1,11 +1,12 @@ import re -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from w3lib.html import remove_tags, replace_entities, replace_escape_chars, get_base_url from scrapy.link import Link from .sgml import SgmlLinkExtractor + linkre = re.compile( "|\s.*?>)(.*?)<[/ ]?a>", re.DOTALL | re.IGNORECASE) diff --git a/scrapy/linkextractors/sgml.py b/scrapy/linkextractors/sgml.py index 8940a4d77..98bed15e9 100644 --- a/scrapy/linkextractors/sgml.py +++ b/scrapy/linkextractors/sgml.py @@ -1,11 +1,11 @@ """ SGMLParser-based Link extractors """ -import six -from six.moves.urllib.parse import urljoin import warnings +from urllib.parse import urljoin from sgmllib import SGMLParser +import six from w3lib.url import safe_url_string, canonicalize_url from w3lib.html import strip_html5_whitespace diff --git a/scrapy/mail.py b/scrapy/mail.py index d24de2212..891bb5e09 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -3,21 +3,21 @@ Mail sending helpers See documentation in docs/topics/email.rst """ -from io import BytesIO import logging - -from email.utils import COMMASPACE, formatdate -from six.moves.email_mime_multipart import MIMEMultipart -from six.moves.email_mime_text import MIMEText -from six.moves.email_mime_base import MIMEBase -from email.mime.nonmultipart import MIMENonMultipart from email import encoders as Encoders +from email.mime.base import MIMEBase +from email.mime.multipart import MIMEMultipart +from email.mime.nonmultipart import MIMENonMultipart +from email.mime.text import MIMEText +from email.utils import COMMASPACE, formatdate +from io import BytesIO from twisted.internet import defer, reactor, ssl from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes + logger = logging.getLogger(__name__) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 8d74c5011..432d4c182 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -5,17 +5,17 @@ See documentation in topics/media-pipeline.rst """ import functools import hashlib -from io import BytesIO +import logging import mimetypes import os import os.path import time -import logging -from email.utils import parsedate_tz, mktime_tz -from six.moves.urllib.parse import urlparse from collections import defaultdict -import six +from email.utils import parsedate_tz, mktime_tz +from io import BytesIO +from urllib.parse import urlparse +import six from twisted.internet import defer, threads from scrapy.pipelines.media import MediaPipeline diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 95a8c09b8..397924110 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -53,7 +53,7 @@ class RobotParser(with_metaclass(ABCMeta)): class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body, spider): - from six.moves.urllib_robotparser import RobotFileParser + from urllib.robotparser import RobotFileParser self.spider = spider robotstxt_body = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True) self.rp = RobotFileParser() diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c76e4d5a2..dce2b3598 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -2,8 +2,8 @@ RefererMiddleware: populates Request referer field, based on the Response which originated it. """ -from six.moves.urllib.parse import urlparse import warnings +from urllib.parse import urlparse from w3lib.url import safe_url_string diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 30cc926e5..d5d3be67e 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -3,7 +3,7 @@ Scheduler queues """ import marshal -from six.moves import cPickle as pickle +import pickle from queuelib import queue diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 5bbda6e27..cdbe21942 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -1,5 +1,6 @@ import random -from six.moves.urllib.parse import urlencode +from urllib.parse import urlencode + from twisted.web.server import Site from twisted.web.resource import Resource from twisted.internet import reactor diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index b3fd0a497..a0a47e473 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -1,9 +1,9 @@ import argparse import warnings from shlex import split +from http.cookies import SimpleCookie +from urllib.parse import urlparse -from six.moves.http_cookies import SimpleCookie -from six.moves.urllib.parse import urlparse from six import string_types, iteritems from w3lib.http import basic_auth_header diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py index b4c929b0e..54ffe086d 100644 --- a/scrapy/utils/httpobj.py +++ b/scrapy/utils/httpobj.py @@ -1,8 +1,8 @@ """Helper functions for scrapy.http objects (Request, Response)""" import weakref +from urllib.parse import urlparse -from six.moves.urllib.parse import urlparse _urlparse_cache = weakref.WeakKeyDictionary() def urlparse_cached(request_or_response): diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 1cbda141a..f28c2eaa1 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -1,5 +1,5 @@ import os -from six.moves import cPickle as pickle +import pickle import warnings from importlib import import_module @@ -7,8 +7,8 @@ from os.path import join, dirname, abspath, isabs, exists from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env from scrapy.settings import Settings -from scrapy.exceptions import NotConfigured -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning + ENVVAR = 'SCRAPY_SETTINGS_MODULE' DATADIR_CFG_SECTION = 'datadir' diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 974abaeb1..cb8ac3c8e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -65,7 +65,7 @@ def is_listlike(x): True >>> is_listlike((x for x in range(3))) True - >>> is_listlike(six.moves.xrange(5)) + >>> is_listlike(range(5)) True """ return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes)) diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 45f1ef17e..c5e877acd 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -5,13 +5,13 @@ scrapy.http.Request objects import hashlib import weakref -from six.moves.urllib.parse import urlunparse +from urllib.parse import urlunparse from w3lib.http import basic_auth_header -from scrapy.utils.python import to_bytes, to_unicode - from w3lib.url import canonicalize_url + from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes, to_unicode _fingerprint_cache = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py index 4742b3e13..2f10cf4de 100644 --- a/scrapy/utils/sitemap.py +++ b/scrapy/utils/sitemap.py @@ -5,8 +5,9 @@ Note: The main purpose of this module is to provide support for the SitemapSpider, its API is subject to change without notice. """ +from urllib.parse import urljoin + import lxml.etree -from six.moves.urllib.parse import urljoin class Sitemap(object): diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index 05d06d53b..6f5c21624 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -1,4 +1,4 @@ -from six.moves.urllib.parse import urljoin +from urllib.parse import urljoin from twisted.internet import reactor from twisted.web import server, resource, static, util diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index b3a4be007..4b48868fe 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,7 +7,7 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath import re -from six.moves.urllib.parse import (ParseResult, urldefrag, urlparse, urlunparse) +from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse # scrapy.utils.url was moved to w3lib.url and import * ensures this # move doesn't break old code