diff --git a/scrapy/contrib_exp/adaptors/__init__.py b/scrapy/contrib_exp/adaptors/__init__.py
index bd78e6ab8..2b6481183 100644
--- a/scrapy/contrib_exp/adaptors/__init__.py
+++ b/scrapy/contrib_exp/adaptors/__init__.py
@@ -1,5 +1,5 @@
from scrapy.contrib_exp.adaptors.extraction import extract, ExtractImageLinks
-from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, remove_escape, unquote
+from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, replace_escape, unquote
from scrapy.contrib_exp.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, delist, Regex
from scrapy.contrib_exp.adaptors.date import to_date
diff --git a/scrapy/contrib_exp/adaptors/markup.py b/scrapy/contrib_exp/adaptors/markup.py
index 13a6f16de..d7d4eacec 100644
--- a/scrapy/contrib_exp/adaptors/markup.py
+++ b/scrapy/contrib_exp/adaptors/markup.py
@@ -1,7 +1,6 @@
import re
-from scrapy.utils.markup import replace_tags, remove_escape_chars, unquote_markup
+from scrapy.utils.markup import replace_tags, replace_escape_chars, unquote_markup
from scrapy.utils.python import str_to_unicode
-from scrapy.item.adaptors import adaptize
def remove_tags(tags=()):
"""
@@ -30,8 +29,7 @@ def remove_root(value):
value = m.group(1)
return str_to_unicode(value)
-
-def remove_escape(which_ones=('\n','\t','\r'), replace_by=u''):
+def replace_escape(which_ones=('\n', '\t', '\r'), replace_by=u''):
"""
Factory that returns an adaptor for removing/replacing each escape
character in the `wich_ones` parameter found in the given value.
@@ -43,10 +41,9 @@ def remove_escape(which_ones=('\n','\t','\r'), replace_by=u''):
Output: unicode
"""
- def _remove_escape(value):
- return remove_escape_chars(value, which_ones, replace_by)
- return _remove_escape
-
+ def _replace_escape(value):
+ return replace_escape_chars(value, which_ones, replace_by)
+ return _replace_escape
def unquote(keep=None):
"""
diff --git a/scrapy/tests/test_utils_markup.py b/scrapy/tests/test_utils_markup.py
index 06df84a2b..95fb2a50b 100644
--- a/scrapy/tests/test_utils_markup.py
+++ b/scrapy/tests/test_utils_markup.py
@@ -2,7 +2,7 @@
import unittest
from scrapy.utils.markup import remove_entities, replace_tags, remove_comments
-from scrapy.utils.markup import remove_tags_with_content, remove_escape_chars, remove_tags
+from scrapy.utils.markup import remove_tags_with_content, replace_escape_chars, remove_tags
from scrapy.utils.markup import unquote_markup
class UtilsMarkupTest(unittest.TestCase):
@@ -99,20 +99,23 @@ class UtilsMarkupTest(unittest.TestCase):
self.assertEqual(remove_tags_with_content(u'not will removedi will removed', which_ones=('i',)),
u'not will removed')
- def test_remove_escape_chars(self):
+ def test_replace_escape_chars(self):
# make sure it always return unicode
- assert isinstance(remove_escape_chars('no ec'), unicode)
- assert isinstance(remove_escape_chars('no ec', which_ones=('\n','\t',)), unicode)
+ assert isinstance(replace_escape_chars('no ec'), unicode)
+ assert isinstance(replace_escape_chars('no ec', replace_by='str'), unicode)
+ assert isinstance(replace_escape_chars('no ec', which_ones=('\n','\t',)), unicode)
# text without escape chars
- self.assertEqual(remove_escape_chars(u'no ec'), u'no ec')
- self.assertEqual(remove_escape_chars(u'no ec', which_ones=('\n',)), u'no ec')
+ self.assertEqual(replace_escape_chars(u'no ec'), u'no ec')
+ self.assertEqual(replace_escape_chars(u'no ec', which_ones=('\n',)), u'no ec')
# text with escape chars
- self.assertEqual(remove_escape_chars(u'escape\n\n'), u'escape')
- self.assertEqual(remove_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
- self.assertEqual(remove_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
- self.assertEqual(remove_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ')
+ self.assertEqual(replace_escape_chars(u'escape\n\n'), u'escape')
+ self.assertEqual(replace_escape_chars(u'escape\n', which_ones=('\t',)), u'escape\n')
+ self.assertEqual(replace_escape_chars(u'escape\tchars\n', which_ones=('\t')), 'escapechars\n')
+ self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=' '), 'escape chars ')
+ self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by=u'\xa3'), u'escape\xa3chars\xa3')
+ self.assertEqual(replace_escape_chars(u'escape\tchars\n', replace_by='\xc2\xa3'), u'escape\xa3chars\xa3')
def test_unquote_markup(self):
sample_txt1 = u"""hi, this is sample text with entities: & ©
diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py
index fd6e1dc25..dc2f87e5f 100644
--- a/scrapy/utils/markup.py
+++ b/scrapy/utils/markup.py
@@ -105,7 +105,7 @@ def remove_tags_with_content(text, which_ones=()):
return text
-def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
+def replace_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
""" Remove escape chars. Default : \\n, \\t, \\r
which_ones -- is a tuple of which escape chars we want to remove.
@@ -118,6 +118,9 @@ def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_by=u''):
text = text.replace(ec, str_to_unicode(replace_by))
return str_to_unicode(text)
+# FIXME: backwards compatibility - should be removed before 0.7 release
+remove_escape_chars = replace_escape_chars
+
def unquote_markup(text, keep=(), remove_illegal=True):
"""
This function receives markup as a text (always a unicode string or a utf-8 encoded string) and does the following: