RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

replaced use of deprecated w3lib.url.urljoin_rfc by stdlib urlparse.urljoin

This commit is contained in:
Pablo Hoffman 2011-09-02 19:09:21 -03:00
parent a1dbc62b45
commit c8d30c6ffa
5 changed files with 22 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -1,4 +1,4 @@
from w3lib.url import urljoin_rfc
from urlparse import urljoin
from scrapy import log
from scrapy.http import HtmlResponse
@ -22,19 +22,19 @@ class RedirectMiddleware(object):
return response
if request.method.upper() == 'HEAD':
if response.status in [301, 302, 303, 307] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected_url = urljoin(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
else:
return response
if response.status in [302, 303] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected_url = urljoin(request.url, response.headers['location'])
redirected = self._redirect_request_using_get(request, redirected_url)
return self._redirect(redirected, request, spider, response.status)
if response.status in [301, 307] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected_url = urljoin(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)

9
scrapy/contrib/linkextractors/htmlparser.py
View File

@ -3,8 +3,9 @@ HTMLParser-based link extractor
"""
from HTMLParser import HTMLParser
from urlparse import urljoin
from w3lib.url import safe_url_string, urljoin_rfc
from w3lib.url import safe_url_string
from scrapy.link import Link
from scrapy.utils.python import unique as unique_list
@ -27,9 +28,11 @@ class HtmlParserLinkExtractor(HTMLParser):
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
ret = []
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
if isinstance(link.url, unicode):
link.url = link.url.encode(response_encoding)
link.url = urljoin(base_url, link.url)
link.url = safe_url_string(link.url, response_encoding)
link.text = link.text.decode(response_encoding)
ret.append(link)

6
scrapy/contrib/linkextractors/image.py
View File

@ -3,7 +3,7 @@ This module implements the HtmlImageLinkExtractor for extracting
image links only.
"""
from w3lib.url import urljoin_rfc
from urlparse import urljoin
from scrapy.link import Link
from scrapy.utils.url import canonicalize_url
from scrapy.utils.python import unicode_to_str, flatten
@ -51,7 +51,7 @@ class HTMLImageLinkExtractor(object):
def extract_links(self, response):
xs = HtmlXPathSelector(response)
base_url = xs.select('//base/@href').extract()
base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url
base_url = urljoin(response.url, base_url[0].encode(response.encoding)) if base_url else response.url
links = []
for location in self.locations:
@ -67,7 +67,7 @@ class HTMLImageLinkExtractor(object):
seen, ret = set(), []
for link in links:
link.url = urljoin_rfc(base_url, link.url, response.encoding)
link.url = urljoin(base_url, link.url)
if self.unique:
if link.url in seen:
continue

6
scrapy/contrib/linkextractors/regex.py
View File

@ -1,6 +1,6 @@
import re
from urlparse import urljoin
from w3lib.url import urljoin_rfc
from w3lib.html import remove_tags, remove_entities, replace_escape_chars
from scrapy.link import Link
@ -18,9 +18,9 @@ class RegexLinkExtractor(SgmlLinkExtractor):
"""High performant link extractor"""
def _extract_links(self, response_text, response_url, response_encoding):
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding))))
clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding))))
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()
links_text = linkre.findall(response_text)

10
scrapy/contrib/linkextractors/sgml.py
View File

@ -3,9 +3,9 @@ SGMLParser-based Link extractors
"""
import re
from urlparse import urlparse
from urlparse import urlparse, urljoin
from w3lib.url import safe_url_string, urljoin_rfc
from w3lib.url import safe_url_string
from scrapy.selector import HtmlXPathSelector
from scrapy.link import Link
@ -33,9 +33,11 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
ret = []
if base_url is None:
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
for link in self.links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
if isinstance(link.url, unicode):
link.url = link.url.encode(response_encoding)
link.url = urljoin(base_url, link.url)
link.url = safe_url_string(link.url, response_encoding)
link.text = str_to_unicode(link.text, response_encoding, errors='replace')
ret.append(link)