RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #561 from redapple/regexlx-encoding 

RegexLinkExtractor: encode URL unicode value when creating Links
This commit is contained in:
Daniel Graña 2014-01-24 07:33:23 -08:00
parent 2d60f86084 f87859a627
commit b14dabb281
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/linkextractors/regex.py
View File

@ -7,7 +7,7 @@ from scrapy.link import Link
from .sgml import SgmlLinkExtractor
linkre = re.compile(
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
"<a\s.*?href=(\"[.#]+?\"|\'[.#]+?\'|[^\s]+?)(>|\s.*?>)(.*?)<[/ ]?a>",
re.DOTALL | re.IGNORECASE)
def clean_link(link_text):
@ -25,6 +25,8 @@ class RegexLinkExtractor(SgmlLinkExtractor):
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()
links_text = linkre.findall(response_text)
urlstext = set([(clean_url(url), clean_text(text)) for url, _, text in links_text])
urlstext = set([(clean_url(url).encode(response_encoding), clean_text(text))
for url, _, text in links_text])
return [Link(url, text) for url, text in urlstext]