diff --git a/scrapy/utils/decorator.py b/scrapy/utils/decorator.py index 623736577..38bee1a6c 100644 --- a/scrapy/utils/decorator.py +++ b/scrapy/utils/decorator.py @@ -5,21 +5,27 @@ from twisted.internet import defer, threads from scrapy.exceptions import ScrapyDeprecationWarning + def deprecated(use_instead=None): """This is a decorator which can be used to mark functions as deprecated. It will result in a warning being emitted when the function is used.""" - def wrapped(func): + def deco(func): @wraps(func) - def new_func(*args, **kwargs): + def wrapped(*args, **kwargs): message = "Call to deprecated function %s." % func.__name__ if use_instead: message += " Use %s instead." % use_instead warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2) return func(*args, **kwargs) - return new_func - return wrapped + return wrapped + + if callable(use_instead): + deco = deco(use_instead) + use_instead = None + return deco + def defers(func): """Decorator to make sure a function always returns a deferred""" diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 03e98bbe2..27b117b6d 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -1,11 +1,10 @@ import re, csv from cStringIO import StringIO -from scrapy.http import TextResponse +from scrapy.http import TextResponse, Response from scrapy.selector import Selector from scrapy import log from scrapy.utils.python import re_rsearch, str_to_unicode -from scrapy.utils.response import body_or_str def xmliter(obj, nodename): @@ -19,7 +18,7 @@ def xmliter(obj, nodename): """ HEADER_START_RE = re.compile(r'^(.*?)<\s*%s(?:\s|>)' % nodename, re.S) HEADER_END_RE = re.compile(r'<\s*/%s\s*>' % nodename, re.S) - text = body_or_str(obj) + text = _body_or_str(obj) header_start = re.search(HEADER_START_RE, text) header_start = header_start.group(1).strip() if header_start else '' @@ -49,7 +48,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None): def _getrow(csv_r): return [str_to_unicode(field, encoding) for field in next(csv_r)] - lines = StringIO(body_or_str(obj, unicode=False)) + lines = StringIO(_body_or_str(obj, unicode=False)) if delimiter: csv_r = csv.reader(lines, delimiter=delimiter) else: @@ -67,3 +66,13 @@ def csviter(obj, delimiter=None, headers=None, encoding=None): else: yield dict(zip(headers, row)) + +def _body_or_str(obj, unicode=True): + assert isinstance(obj, (Response, basestring)), \ + "obj must be Response or basestring, not %s" % type(obj).__name__ + if isinstance(obj, Response): + return obj.body_as_unicode() if unicode else obj.body + elif isinstance(obj, str): + return obj.decode('utf-8') if unicode else obj + else: + return obj if unicode else obj.encode('utf-8') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 85d0b60f0..61f43535f 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -13,17 +13,15 @@ from twisted.web import http from twisted.web.http import RESPONSES from w3lib import html -from scrapy.http import Response, HtmlResponse, TextResponse +from scrapy.http import HtmlResponse, TextResponse +from scrapy.utils.decorator import deprecated + + +@deprecated +def body_or_str(*a, **kw): + from scrapy.utils.iterators import _body_or_str + return _body_or_str(*a, **kw) -def body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, basestring)), \ - "obj must be Response or basestring, not %s" % type(obj).__name__ - if isinstance(obj, Response): - return obj.body_as_unicode() if unicode else obj.body - elif isinstance(obj, str): - return obj.decode('utf-8') if unicode else obj - else: - return obj if unicode else obj.encode('utf-8') _baseurl_cache = weakref.WeakKeyDictionary() def get_base_url(response):