RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use w3lib.html.replace_entities() (remove_entities() is deprecated)

This commit is contained in:
Paul Tremberth 2014-08-03 12:02:25 +02:00
parent e62bbf0766
commit 928e7f2924
3 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/contrib/downloadermiddleware/ajaxcrawl.py
View File

@ -84,6 +84,6 @@ def _has_ajaxcrawlable_meta(text):
text = _script_re.sub(u'', text)
text = _noscript_re.sub(u'', text)
text = html.remove_comments(html.remove_entities(text))
text = html.remove_comments(html.replace_entities(text))
return _ajax_crawlable_re.search(text) is not None

4
scrapy/contrib/linkextractors/regex.py
View File

@ -1,7 +1,7 @@
import re
from six.moves.urllib.parse import urljoin
from w3lib.html import remove_tags, remove_entities, replace_escape_chars
from w3lib.html import remove_tags, replace_entities, replace_escape_chars
from scrapy.link import Link
from .sgml import SgmlLinkExtractor
@ -21,7 +21,7 @@ class RegexLinkExtractor(SgmlLinkExtractor):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding))))
clean_url = lambda u: urljoin(base_url, replace_entities(clean_link(u.decode(response_encoding))))
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()
links_text = linkre.findall(response_text)

6
scrapy/utils/misc.py
View File

@ -5,7 +5,7 @@ from importlib import import_module
from pkgutil import iter_modules
import six
from w3lib.html import remove_entities
from w3lib.html import replace_entities
from scrapy.utils.python import flatten
from scrapy.item import BaseItem
@ -94,9 +94,9 @@ def extract_regex(regex, text, encoding='utf-8'):
strings = flatten(strings)
if isinstance(text, unicode):
return [remove_entities(s, keep=['lt', 'amp']) for s in strings]
return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
else:
return [remove_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings]
return [replace_entities(unicode(s, encoding), keep=['lt', 'amp']) for s in strings]
def md5sum(file):