renamed remove_escape_chars to replace_escape_chars (adaptor and function), added more tests to replace_escape_chars, keeping backwards compatibility

scrapy/contrib_exp/adaptors/__init__.py

@@ -1,5 +1,5 @@
 from scrapy.contrib_exp.adaptors.extraction import extract, ExtractImageLinks
-from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, remove_escape, unquote
+from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, replace_escape, unquote
 from scrapy.contrib_exp.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, delist, Regex
 from scrapy.contrib_exp.adaptors.date import to_date
 

scrapy/contrib_exp/adaptors/markup.py

@@ -1,7 +1,6 @@
 import re
-from scrapy.utils.markup import replace_tags, remove_escape_chars, unquote_markup
+from scrapy.utils.markup import replace_tags, replace_escape_chars, unquote_markup
 from scrapy.utils.python import str_to_unicode
-from scrapy.item.adaptors import adaptize
 
 def remove_tags(tags=()):
     """
@@ -30,8 +29,7 @@ def remove_root(value):
         value = m.group(1)
     return str_to_unicode(value)
 
-
-def remove_escape(which_ones=('\n','\t','\r'), replace_by=u''):
+def replace_escape(which_ones=('\n', '\t', '\r'), replace_by=u''):
     """
     Factory that returns an adaptor for removing/replacing each escape
     character in the `wich_ones` parameter found in the given value.
@@ -43,10 +41,9 @@ def remove_escape(which_ones=('\n','\t','\r'), replace_by=u''):
     Output: unicode
     
     """
-    def _remove_escape(value):
-        return remove_escape_chars(value, which_ones, replace_by)
-    return _remove_escape
-
+    def _replace_escape(value):
+        return replace_escape_chars(value, which_ones, replace_by)
+    return _replace_escape
 
 def unquote(keep=None):
     """

scrapy/tests/test_utils_markup.py

@@ -2,7 +2,7 @@
 import unittest
 
 from scrapy.utils.markup import remove_entities, replace_tags, remove_comments
-from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags
+from scrapy.utils.markup import remove_tags_with_content, replace_escape_chars, remove_tags
 from scrapy.utils.markup import unquote_markup
 
 class UtilsMarkupTest(unittest.TestCase):
@@ -99,20 +99,23 @@ class UtilsMarkupTest(unittest.TestCase):
         self.assertEqual(remove_tags_with_content(u'<b>not will removed</b><i>i will removed</i>', which_ones=('i',)),
                          u'<b>not will removed</b>')
 
-    def test_remove_escape_chars(self):
+    def test_replace_escape_chars(self):
         # make sure it always return unicode
-        assert isinstance(remove_escape_chars('no ec'), unicode)
-        assert isinstance(remove_escape_chars('no ec', which_ones=('\n','\t',)), unicode)
+        assert isinstance(replace_escape_chars('no ec'), unicode)
+        assert isinstance(replace_escape_chars('no ec', replace_by='str'), unicode)
+        assert isinstance(replace_escape_chars('no ec', which_ones=('\n','\t',)), unicode)
 
         # text without escape chars
-        self.assertEqual(remove_escape_chars(u'no ec'), u'no ec')
-        self.assertEqual(remove_escape_chars(u'no ec', which_ones=('\n',)), u'no ec')
+        self.assertEqual(replace_escape_chars(u'no ec'), u'no ec')
+        self.assertEqual(replace_escape_chars(u'no ec', which_ones=('\n',)), u'no ec')
 
         # text with escape chars
-        self.assertEqual(remove_escape_chars(u'escape\n\n'), u'escape')
-        self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
-        self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
-        self.assertEqual(remove_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ')
+        self.assertEqual(replace_escape_chars(u'escape\n\n'), u'escape')
+        self.assertEqual(replace_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
+        self.assertEqual(replace_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
+        self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ')
+        self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=u'\xa3'), u'escape\xa3chars\xa3')
+        self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by='\xc2\xa3'), u'escape\xa3chars\xa3')
 
     def test_unquote_markup(self):
         sample_txt1 = u"""<node1>hi, this is sample text with entities: &amp; &copy;

scrapy/utils/markup.py

@@ -105,7 +105,7 @@ def remove_tags_with_content(text, which_ones=()):
     return text
     
 
-def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
+def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
     """ Remove escape chars. Default : \\n, \\t, \\r
 
         which_ones -- is a tuple of which escape chars we want to remove.
@@ -118,6 +118,9 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
         text = text.replace(ec, str_to_unicode(replace_by))
     return str_to_unicode(text)
 
+# FIXME: backwards compatibility - should be removed before 0.7 release
+remove_escape_chars = replace_escape_chars
+
 def unquote_markup(text, keep=(), remove_illegal=True):
     """
     This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following: