diff --git a/extras/scrapy-ws.py b/extras/scrapy-ws.py index 60fdfd4af..b95298d6e 100755 --- a/extras/scrapy-ws.py +++ b/extras/scrapy-ws.py @@ -13,7 +13,7 @@ command). from __future__ import print_function import sys, optparse, urllib, json -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from scrapy.utils.jsonrpc import jsonrpc_client_call, JsonRpcError diff --git a/scrapy/commands/deploy.py b/scrapy/commands/deploy.py index 62b57d3ed..051ce661a 100644 --- a/scrapy/commands/deploy.py +++ b/scrapy/commands/deploy.py @@ -8,7 +8,7 @@ import time import urllib2 import netrc import json -from urlparse import urlparse, urljoin +from six.moves.urllib.parse import urlparse, urljoin from subprocess import Popen, PIPE, check_call from w3lib.form import encode_multipart diff --git a/scrapy/contrib/downloadermiddleware/httpproxy.py b/scrapy/contrib/downloadermiddleware/httpproxy.py index 6a6a049bf..ce09655d0 100644 --- a/scrapy/contrib/downloadermiddleware/httpproxy.py +++ b/scrapy/contrib/downloadermiddleware/httpproxy.py @@ -1,7 +1,7 @@ import base64 from urllib import getproxies, unquote, proxy_bypass from urllib2 import _parse_proxy -from urlparse import urlunparse +from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 1b36a7eae..6a42987e1 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,4 @@ -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from scrapy import log from scrapy.http import HtmlResponse diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index cc9f863f9..f1e8012e7 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -4,7 +4,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. """ -import robotparser +from six.moves.urllib import robotparser from scrapy import signals, log from scrapy.exceptions import NotConfigured, IgnoreRequest diff --git a/scrapy/contrib/feedexport.py b/scrapy/contrib/feedexport.py index bf16cd0f4..941f9633e 100644 --- a/scrapy/contrib/feedexport.py +++ b/scrapy/contrib/feedexport.py @@ -7,7 +7,7 @@ See documentation in docs/topics/feed-exports.rst import sys, os, posixpath from tempfile import TemporaryFile from datetime import datetime -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from ftplib import FTP from zope.interface import Interface, implements diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index 6e3c5071a..fff9eabe6 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -3,7 +3,7 @@ HTMLParser-based link extractor """ from HTMLParser import HTMLParser -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from w3lib.url import safe_url_string diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index efe24b835..b6de74f33 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -3,7 +3,7 @@ Link extractor based on lxml.html """ import re -from urlparse import urlparse, urljoin +from six.moves.urllib.parse import urlparse, urljoin import lxml.etree as etree diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index db2c9db8e..e9d77e618 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -1,5 +1,5 @@ import re -from urlparse import urljoin +from six.moves.urllib.parse import urljoin from w3lib.html import remove_tags, remove_entities, replace_escape_chars diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 4e54e9b7e..9a55c1581 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -1,7 +1,7 @@ """ SGMLParser-based Link extractors """ -from urlparse import urljoin +from six.moves.urllib.parse import urljoin import warnings from sgmllib import SGMLParser diff --git a/scrapy/contrib/pipeline/files.py b/scrapy/contrib/pipeline/files.py index 09a0ca665..8c43982a1 100644 --- a/scrapy/contrib/pipeline/files.py +++ b/scrapy/contrib/pipeline/files.py @@ -7,7 +7,7 @@ import os import os.path import rfc822 import time -import urlparse +from six.moves.urllib.parse import urlparse from collections import defaultdict from cStringIO import StringIO import six @@ -167,7 +167,7 @@ class FilesPipeline(MediaPipeline): if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir scheme = 'file' else: - scheme = urlparse.urlparse(uri).scheme + scheme = urlparse(uri).scheme store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 70c526490..6b5f03bda 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -29,7 +29,7 @@ In case of status 200 request, response.headers will come with two keys: """ import re -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from cStringIO import StringIO from twisted.internet import reactor diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 617a68ea4..455794b14 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -4,7 +4,7 @@ import re from time import time from cStringIO import StringIO -from urlparse import urldefrag +from six.moves.urllib.parse import urldefrag from zope.interface import implements from twisted.internet import defer, reactor, protocol diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 135d8f293..2c6a61b8a 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,5 @@ from time import time -from urlparse import urlparse, urlunparse, urldefrag +from six.moves.urllib.parse import urlparse, urlunparse, urldefrag from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index a4f45a306..53488e1a1 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,8 @@ This module implements the FormRequest class which is a more covenient class See documentation in docs/topics/request-response.rst """ -import urllib, urlparse +import urllib +from six.moves.urllib.parse import urljoin import lxml.html import six from scrapy.http.request import Request @@ -43,7 +44,7 @@ class FormRequest(Request): def _get_form_url(form, url): if url is None: return form.action or form.base_url - return urlparse.urljoin(form.base_url, url) + return urljoin(form.base_url, url) def _urlencode(seq, enc): values = [(unicode_to_str(k, enc), unicode_to_str(v, enc)) diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index d9c85b50a..5badea5e5 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -3,7 +3,7 @@ Common code and definitions used by Link extractors (located in scrapy.contrib.linkextractor). """ import re -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from scrapy.utils.url import url_is_from_any_domain from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension diff --git a/scrapy/tests/test_contrib_feedexport.py b/scrapy/tests/test_contrib_feedexport.py index cf85ea1e7..7a1b3dc8f 100644 --- a/scrapy/tests/test_contrib_feedexport.py +++ b/scrapy/tests/test_contrib_feedexport.py @@ -1,4 +1,5 @@ -import os, urlparse +import os +from six.moves.urllib.parse import urlparse from cStringIO import StringIO from zope.interface.verify import verifyObject @@ -84,7 +85,7 @@ class S3FeedStorageTest(unittest.TestCase): file = storage.open(Spider("default")) file.write("content") yield storage.store(file) - u = urlparse.urlparse(uri) + u = urlparse(uri) key = connect_s3().get_bucket(u.hostname, validate=False).get_key(u.path) self.failUnlessEqual(key.get_contents_as_string(), "content") diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index cabdff9bb..2881b9168 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -11,7 +11,8 @@ module with the ``runserver`` argument:: """ from __future__ import print_function -import sys, os, re, urlparse +import sys, os, re +from six.moves.urllib.parse import urlparse from twisted.internet import reactor, defer from twisted.web import server, static, util @@ -117,7 +118,7 @@ class CrawlerRun(object): return "http://localhost:%s%s" % (self.portno, path) def getpath(self, url): - u = urlparse.urlparse(url) + u = urlparse(url) return u.path def item_scraped(self, item, spider, response): diff --git a/scrapy/tests/test_http_cookies.py b/scrapy/tests/test_http_cookies.py index bead255c5..3d6993491 100644 --- a/scrapy/tests/test_http_cookies.py +++ b/scrapy/tests/test_http_cookies.py @@ -1,4 +1,4 @@ -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from unittest import TestCase from scrapy.http import Request, Response diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index ebcee8c1d..d933df258 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,7 +1,7 @@ import cgi import unittest import xmlrpclib -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, HtmlResponse diff --git a/scrapy/tests/test_spidermiddleware_offsite.py b/scrapy/tests/test_spidermiddleware_offsite.py index 7e0d248dc..31ed9c909 100644 --- a/scrapy/tests/test_spidermiddleware_offsite.py +++ b/scrapy/tests/test_spidermiddleware_offsite.py @@ -5,7 +5,7 @@ from scrapy.spider import Spider from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler -from urlparse import urlparse +from six.moves.urllib.parse import urlparse class TestOffsiteMiddleware(TestCase): diff --git a/scrapy/tests/test_urlparse_monkeypatches.py b/scrapy/tests/test_urlparse_monkeypatches.py index 91d75789c..052dde37f 100644 --- a/scrapy/tests/test_urlparse_monkeypatches.py +++ b/scrapy/tests/test_urlparse_monkeypatches.py @@ -1,4 +1,4 @@ -from urlparse import urlparse +from six.moves.urllib.parse import urlparse import unittest diff --git a/scrapy/tests/test_utils_httpobj.py b/scrapy/tests/test_utils_httpobj.py index e978ae851..4f9f7a370 100644 --- a/scrapy/tests/test_utils_httpobj.py +++ b/scrapy/tests/test_utils_httpobj.py @@ -1,5 +1,5 @@ import unittest -import urlparse +from six.moves.urllib.parse import urlparse from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached @@ -13,7 +13,7 @@ class HttpobjUtilsTest(unittest.TestCase): req1a = urlparse_cached(request1) req1b = urlparse_cached(request1) req2 = urlparse_cached(request2) - urlp = urlparse.urlparse(url) + urlp = urlparse(url) assert req1a == req2 assert req1a == urlp diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 5a646a34e..92c92c057 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -1,6 +1,6 @@ import os import unittest -import urlparse +from six.moves.urllib.parse import urlparse from scrapy.http import Response, TextResponse, HtmlResponse from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh @@ -24,7 +24,7 @@ class ResponseUtilsTest(unittest.TestCase): url = "http:///www.example.com/some/page.html" body = "