diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py index 051ce661a..b7d989f69 100644 --- a/scrapy/commands/deploy.py +++ b/scrapy/commands/deploy.py @@ -5,10 +5,11 @@ import glob import tempfile import shutil import time -import urllib2 import netrc import json +from six.moves.urllib import request from six.moves.urllib.parse import urlparse, urljoin +from six.moves.urllib.error import HTTPError, URLError from subprocess import Popen, PIPE, check_call from w3lib.form import encode_multipart @@ -69,7 +70,7 @@ class Command(ScrapyCommand): except ImportError: raise UsageError("setuptools not installed") - urllib2.install_opener(urllib2.build_opener(HTTPRedirectHandler)) + request.install_opener(request.build_opener(HTTPRedirectHandler)) if opts.list_targets: for name, target in _get_targets().items(): @@ -78,9 +79,9 @@ class Command(ScrapyCommand): if opts.list_projects: target = _get_target(opts.list_projects) - req = urllib2.Request(_url(target, 'listprojects.json')) + req = request.Request(_url(target, 'listprojects.json')) _add_auth_header(req, target) - f = urllib2.urlopen(req) + f = request.urlopen(req) projects = json.loads(f.read())['projects'] print(os.linesep.join(projects)) return @@ -188,7 +189,7 @@ def _upload_egg(target, eggpath, project, version): 'Content-Type': 'multipart/form-data; boundary=%s' % boundary, 'Content-Length': str(len(body)), } - req = urllib2.Request(url, body, headers) + req = request.Request(url, body, headers) _add_auth_header(req, target) _log('Deploying to project "%s" in %s' % (project, url)) return _http_post(req) @@ -207,14 +208,14 @@ def _add_auth_header(request, target): def _http_post(request): try: - f = urllib2.urlopen(request) + f = request.urlopen(request) _log("Server response (%s):" % f.code) print(f.read()) return True - except urllib2.HTTPError as e: + except HTTPError as e: _log("Deploy failed (%s):" % e.code) print(e.read()) - except urllib2.URLError as e: + except URLError as e: _log("Deploy failed: %s" % e) def _build_egg(): @@ -237,12 +238,12 @@ def _create_default_setup_py(**kwargs): f.write(_SETUP_PY_TEMPLATE % kwargs) -class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): +class HTTPRedirectHandler(request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): newurl = newurl.replace(' ', '%20') if code in (301, 307): - return urllib2.Request(newurl, + return request.Request(newurl, data=req.get_data(), headers=req.headers, origin_req_host=req.get_origin_req_host(), @@ -250,9 +251,9 @@ class HTTPRedirectHandler(urllib2.HTTPRedirectHandler): elif code in (302, 303): newheaders = dict((k, v) for k, v in req.headers.items() if k.lower() not in ("content-length", "content-type")) - return urllib2.Request(newurl, + return request.Request(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) else: - raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) + raise HTTPError(req.get_full_url(), code, msg, headers, fp) diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py index ce09655d0..48a877c91 100644 --- a/scrapy/contrib/downloadermiddleware/httpproxy.py +++ b/scrapy/contrib/downloadermiddleware/httpproxy.py @@ -1,6 +1,10 @@ import base64 -from urllib import getproxies, unquote, proxy_bypass -from urllib2 import _parse_proxy +from six.moves.urllib.request import getproxies, proxy_bypass +from six.moves.urllib.parse import unquote +try: + from urllib2 import _parse_proxy +except ImportError: + from urllib.request import _parse_proxy from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 02e77ce29..a4695f1a2 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,8 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -import urllib -from six.moves.urllib.parse import urljoin +from six.moves.urllib.parse import urljoin, urlencode import lxml.html import six from scrapy.http.request import Request @@ -52,7 +51,7 @@ def _urlencode(seq, enc): values = [(unicode_to_str(k, enc), unicode_to_str(v, enc)) for k, vs in seq for v in (vs if hasattr(vs, '__iter__') else [vs])] - return urllib.urlencode(values, doseq=1) + return urlencode(values, doseq=1) def _get_form(response, formname, formnumber, formxpath): diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 36490a39d..5c645aaf0 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -7,8 +7,8 @@ to the w3lib.url module. Always import those from there instead. """ import posixpath from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag, - urlparse, parse_qsl) -import urllib + urlparse, parse_qsl, urlencode, + unquote) # scrapy.utils.url was moved to w3lib.url and import * ensures this move doesn't break old code from w3lib.url import * @@ -56,7 +56,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, scheme, netloc, path, params, query, fragment = parse_url(url) keyvals = parse_qsl(query, keep_blank_values) keyvals.sort() - query = urllib.urlencode(keyvals) + query = urlencode(keyvals) path = safe_url_string(_unquotepath(path)) or '/' fragment = '' if not keep_fragments else fragment return urlunparse((scheme, netloc.lower(), path, params, query, fragment)) @@ -65,7 +65,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, def _unquotepath(path): for reserved in ('2f', '2F', '3f', '3F'): path = path.replace('%' + reserved, '%25' + reserved.upper()) - return urllib.unquote(path) + return unquote(path) def parse_url(url, encoding=None): diff --git a/tests/mockserver.py b/tests/mockserver.py index b73208c5c..d2ba92485 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,6 +1,7 @@ from __future__ import print_function -import sys, time, random, urllib, os, json +import sys, time, random, os, json import six +from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource @@ -73,7 +74,7 @@ class Follow(LeafResource): args = request.args.copy() for nl in nlist: args["n"] = [str(nl)] - argstr = urllib.urlencode(args, doseq=True) + argstr = urlencode(args, doseq=True) s += "follow %d
" % (argstr, nl) s += """""" request.write(s)