From c8d30c6ffa6185bb98ffd4f74c64a9b1d265f852 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 2 Sep 2011 19:09:21 -0300 Subject: [PATCH] replaced use of deprecated w3lib.url.urljoin_rfc by stdlib urlparse.urljoin --- scrapy/contrib/downloadermiddleware/redirect.py | 8 ++++---- scrapy/contrib/linkextractors/htmlparser.py | 9 ++++++--- scrapy/contrib/linkextractors/image.py | 6 +++--- scrapy/contrib/linkextractors/regex.py | 6 +++--- scrapy/contrib/linkextractors/sgml.py | 10 ++++++---- 5 files changed, 22 insertions(+), 17 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index c8a401d0f..02e402483 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -1,4 +1,4 @@ -from w3lib.url import urljoin_rfc +from urlparse import urljoin from scrapy import log from scrapy.http import HtmlResponse @@ -22,19 +22,19 @@ class RedirectMiddleware(object): return response if request.method.upper() == 'HEAD': if response.status in [301, 302, 303, 307] and 'Location' in response.headers: - redirected_url = urljoin_rfc(request.url, response.headers['location']) + redirected_url = urljoin(request.url, response.headers['location']) redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) else: return response if response.status in [302, 303] and 'Location' in response.headers: - redirected_url = urljoin_rfc(request.url, response.headers['location']) + redirected_url = urljoin(request.url, response.headers['location']) redirected = self._redirect_request_using_get(request, redirected_url) return self._redirect(redirected, request, spider, response.status) if response.status in [301, 307] and 'Location' in response.headers: - redirected_url = urljoin_rfc(request.url, response.headers['location']) + redirected_url = urljoin(request.url, response.headers['location']) redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index 0f979d3c1..aa68ce52f 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -3,8 +3,9 @@ HTMLParser-based link extractor """ from HTMLParser import HTMLParser +from urlparse import urljoin -from w3lib.url import safe_url_string, urljoin_rfc +from w3lib.url import safe_url_string from scrapy.link import Link from scrapy.utils.python import unique as unique_list @@ -27,9 +28,11 @@ class HtmlParserLinkExtractor(HTMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url + base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in links: - link.url = urljoin_rfc(base_url, link.url, response_encoding) + if isinstance(link.url, unicode): + link.url = link.url.encode(response_encoding) + link.url = urljoin(base_url, link.url) link.url = safe_url_string(link.url, response_encoding) link.text = link.text.decode(response_encoding) ret.append(link) diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index cac87ec1d..741060769 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -3,7 +3,7 @@ This module implements the HtmlImageLinkExtractor for extracting image links only. """ -from w3lib.url import urljoin_rfc +from urlparse import urljoin from scrapy.link import Link from scrapy.utils.url import canonicalize_url from scrapy.utils.python import unicode_to_str, flatten @@ -51,7 +51,7 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) base_url = xs.select('//base/@href').extract() - base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url + base_url = urljoin(response.url, base_url[0].encode(response.encoding)) if base_url else response.url links = [] for location in self.locations: @@ -67,7 +67,7 @@ class HTMLImageLinkExtractor(object): seen, ret = set(), [] for link in links: - link.url = urljoin_rfc(base_url, link.url, response.encoding) + link.url = urljoin(base_url, link.url) if self.unique: if link.url in seen: continue diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 9c349ad3a..fc60488c5 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -1,6 +1,6 @@ import re +from urlparse import urljoin -from w3lib.url import urljoin_rfc from w3lib.html import remove_tags, remove_entities, replace_escape_chars from scrapy.link import Link @@ -18,9 +18,9 @@ class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" def _extract_links(self, response_text, response_url, response_encoding): - base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url + base_url = urljoin(response_url, self.base_url) if self.base_url else response_url - clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding)))) + clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip() links_text = linkre.findall(response_text) diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 4412ef75e..016f2d8fd 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -3,9 +3,9 @@ SGMLParser-based Link extractors """ import re -from urlparse import urlparse +from urlparse import urlparse, urljoin -from w3lib.url import safe_url_string, urljoin_rfc +from w3lib.url import safe_url_string from scrapy.selector import HtmlXPathSelector from scrapy.link import Link @@ -33,9 +33,11 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): ret = [] if base_url is None: - base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url + base_url = urljoin(response_url, self.base_url) if self.base_url else response_url for link in self.links: - link.url = urljoin_rfc(base_url, link.url, response_encoding) + if isinstance(link.url, unicode): + link.url = link.url.encode(response_encoding) + link.url = urljoin(base_url, link.url) link.url = safe_url_string(link.url, response_encoding) link.text = str_to_unicode(link.text, response_encoding, errors='replace') ret.append(link)