RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

force replace_by to be unicode in remove_escape_chars, added tests

This commit is contained in:
Ismael Carnales 2009-05-07 15:44:38 +00:00
parent 7eb79488aa
commit 1b3d40e639
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/tests/test_utils_markup.py
View File

@ -112,6 +112,7 @@ class UtilsMarkupTest(unittest.TestCase):
self.assertEqual(remove_escape_chars(u'escape\n\n'), u'escape')
self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
self.assertEqual(remove_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ')
def test_unquote_markup(self):
sample_txt1 = u"""<node1>hi, this is sample text with entities: &amp; &copy;

4
scrapy/utils/markup.py
View File

@ -105,14 +105,14 @@ def remove_tags_with_content(text, which_ones=()):
return text
def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_str=u''):
def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
""" Remove escape chars. Default : \\n, \\t, \\r
which_ones -- is a tuple of which escape chars we want to remove.
By default removes \n, \t, \r.
"""
for ec in which_ones:
text = text.replace(ec, replace_str)
text = text.replace(ec, str_to_unicode(replace_by))
return str_to_unicode(text)
def unquote_markup(text, keep=(), remove_illegal=True):