diff --git a/scrapy/contrib_exp/adaptors/__init__.py b/scrapy/contrib_exp/adaptors/__init__.py index 8bbc50d6f..bd78e6ab8 100644 --- a/scrapy/contrib_exp/adaptors/__init__.py +++ b/scrapy/contrib_exp/adaptors/__init__.py @@ -1,5 +1,5 @@ from scrapy.contrib_exp.adaptors.extraction import extract, ExtractImageLinks -from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, unquote +from scrapy.contrib_exp.adaptors.markup import remove_tags, remove_root, remove_escape, unquote from scrapy.contrib_exp.adaptors.misc import to_unicode, clean_spaces, strip, drop_empty, delist, Regex from scrapy.contrib_exp.adaptors.date import to_date diff --git a/scrapy/contrib_exp/adaptors/markup.py b/scrapy/contrib_exp/adaptors/markup.py index ac58d4c15..b8f29d3ed 100644 --- a/scrapy/contrib_exp/adaptors/markup.py +++ b/scrapy/contrib_exp/adaptors/markup.py @@ -1,5 +1,5 @@ import re -from scrapy.utils.markup import replace_tags, unquote_markup +from scrapy.utils.markup import replace_tags, remove_escape_chars, unquote_markup from scrapy.utils.python import str_to_unicode from scrapy.item.adaptors import adaptize @@ -30,6 +30,24 @@ def remove_root(value): value = m.group(1) return str_to_unicode(value) + +def remove_escape(which_ones=('\n','\t','\r'), replace_str=u''): + """ + Factory that returns an adaptor for removing/replacing each escape + character in the `wich_ones` parameter found in the given value. + + If `replace_str` is given, escape characters are replaced by that + string, else they're removed. + + Input: string/unicode + Output: unicode + + """ + def _remove_escape(value): + return remove_escape_chars(value, which_ones, replace_str) + return _remove_escape + + def unquote(keep=None): """ This factory returns an adaptor that diff --git a/scrapy/utils/markup.py b/scrapy/utils/markup.py index d008456ed..09ef64192 100644 --- a/scrapy/utils/markup.py +++ b/scrapy/utils/markup.py @@ -105,14 +105,14 @@ def remove_tags_with_content(text, which_ones=()): return text -def remove_escape_chars(text, which_ones=('\n','\t','\r')): +def remove_escape_chars(text, which_ones=('\n','\t','\r'), replace_str=u''): """ Remove escape chars. Default : \\n, \\t, \\r which_ones -- is a tuple of which escape chars we want to remove. By default removes \n, \t, \r. """ for ec in which_ones: - text = text.replace(ec, u'') + text = text.replace(ec, replace_str) return str_to_unicode(text) def unquote_markup(text, keep=(), remove_illegal=True):