RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extract links with safe_url_string 

canonicalize_url changes links in undesirable ways.
This commit is contained in:
Adrián Chaves 2020-02-10 21:57:21 +01:00
parent 35723d76c0
commit 61e74bac76
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/linkextractors/lxmlhtml.py
View File

@ -5,7 +5,7 @@ from urllib.parse import urljoin
import lxml.etree as etree
from w3lib.html import strip_html5_whitespace
from w3lib.url import canonicalize_url
from w3lib.url import canonicalize_url, safe_url_string
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object):
url = self.process_attr(attr_val)
if url is None:
continue
url = canonicalize_url(url, encoding=response_encoding)
url = safe_url_string(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',