diff --git a/scrapy/contrib_exp/adaptors/__init__.py b/scrapy/contrib_exp/adaptors/__init__.py index bd78e6ab8..2b6481183 100644 --- a/scrapy/contrib_exp/adaptors/__init__.py +++ b/scrapy/contrib_exp/adaptors/__init__.py @@ -1,5 +1,5 @@ from scrapy.contrib_exp.adaptors.extraction import extract, ExtractImageLinks -from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, remove_escape, unquote +from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, replace_escape, unquote from scrapy.contrib_exp.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, delist, Regex from scrapy.contrib_exp.adaptors.date import to_date diff --git a/scrapy/contrib_exp/adaptors/markup.py b/scrapy/contrib_exp/adaptors/markup.py index 13a6f16de..d7d4eacec 100644 --- a/scrapy/contrib_exp/adaptors/markup.py +++ b/scrapy/contrib_exp/adaptors/markup.py @@ -1,7 +1,6 @@ import re -from scrapy.utils.markup import replace_tags, remove_escape_chars, unquote_markup +from scrapy.utils.markup import replace_tags, replace_escape_chars, unquote_markup from scrapy.utils.python import str_to_unicode -from scrapy.item.adaptors import adaptize def remove_tags(tags=()): """ @@ -30,8 +29,7 @@ def remove_root(value): value = m.group(1) return str_to_unicode(value) - -def remove_escape(which_ones=('\n','\t','\r'), replace_by=u''): +def replace_escape(which_ones=('\n', '\t', '\r'), replace_by=u''): """ Factory that returns an adaptor for removing/replacing each escape character in the `wich_ones` parameter found in the given value. @@ -43,10 +41,9 @@ def remove_escape(which_ones=('\n','\t','\r'), replace_by=u''): Output: unicode """ - def _remove_escape(value): - return remove_escape_chars(value, which_ones, replace_by) - return _remove_escape - + def _replace_escape(value): + return replace_escape_chars(value, which_ones, replace_by) + return _replace_escape def unquote(keep=None): """ diff --git a/scrapy/tests/test_utils_markup.py b/scrapy/tests/test_utils_markup.py index 06df84a2b..95fb2a50b 100644 --- a/scrapy/tests/test_utils_markup.py +++ b/scrapy/tests/test_utils_markup.py @@ -2,7 +2,7 @@ import unittest from scrapy.utils.markup import remove_entities, replace_tags, remove_comments -from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags +from scrapy.utils.markup import remove_tags_with_content, replace_escape_chars, remove_tags from scrapy.utils.markup import unquote_markup class UtilsMarkupTest(unittest.TestCase): @@ -99,20 +99,23 @@ class UtilsMarkupTest(unittest.TestCase): self.assertEqual(remove_tags_with_content(u'not will removedi will removed', which_ones=('i',)), u'not will removed') - def test_remove_escape_chars(self): + def test_replace_escape_chars(self): # make sure it always return unicode - assert isinstance(remove_escape_chars('no ec'), unicode) - assert isinstance(remove_escape_chars('no ec', which_ones=('\n','\t',)), unicode) + assert isinstance(replace_escape_chars('no ec'), unicode) + assert isinstance(replace_escape_chars('no ec', replace_by='str'), unicode) + assert isinstance(replace_escape_chars('no ec', which_ones=('\n','\t',)), unicode) # text without escape chars - self.assertEqual(remove_escape_chars(u'no ec'), u'no ec') - self.assertEqual(remove_escape_chars(u'no ec', which_ones=('\n',)), u'no ec') + self.assertEqual(replace_escape_chars(u'no ec'), u'no ec') + self.assertEqual(replace_escape_chars(u'no ec', which_ones=('\n',)), u'no ec') # text with escape chars - self.assertEqual(remove_escape_chars(u'escape\n\n'), u'escape') - self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n') - self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n') - self.assertEqual(remove_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ') + self.assertEqual(replace_escape_chars(u'escape\n\n'), u'escape') + self.assertEqual(replace_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n') + self.assertEqual(replace_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n') + self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ') + self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=u'\xa3'), u'escape\xa3chars\xa3') + self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by='\xc2\xa3'), u'escape\xa3chars\xa3') def test_unquote_markup(self): sample_txt1 = u"""hi, this is sample text with entities: & © diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index fd6e1dc25..dc2f87e5f 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -105,7 +105,7 @@ def remove_tags_with_content(text, which_ones=()): return text -def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): +def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): """ Remove escape chars. Default : \\n, \\t, \\r which_ones -- is a tuple of which escape chars we want to remove. @@ -118,6 +118,9 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''): text = text.replace(ec, str_to_unicode(replace_by)) return str_to_unicode(text) +# FIXME: backwards compatibility - should be removed before 0.7 release +remove_escape_chars = replace_escape_chars + def unquote_markup(text, keep=(), remove_illegal=True): """ This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following: