diff --git a/conftest.py b/conftest.py
index b0ac1badd..8b4faf8fc 100644
--- a/conftest.py
+++ b/conftest.py
@@ -34,7 +34,7 @@ if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15,
if six.PY3:
for line in open('tests/py3-ignores.txt'):
file_path = line.strip()
- if len(file_path) > 0 and file_path[0] != '#':
+ if file_path and file_path[0] != '#':
collect_ignore.append(file_path)
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 0837610d0..dd0c6216b 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -47,7 +47,7 @@ Set stat value::
Increment stat value::
- stats.inc_value('pages_crawled')
+ stats.inc_value('custom_count')
Set stat value only if greater than previous::
@@ -59,13 +59,13 @@ Set stat value only if lower than previous::
Get stat value::
- >>> stats.get_value('pages_crawled')
- 8
+ >>> stats.get_value('custom_count')
+ 1
Get all stats::
>>> stats.get_stats()
- {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
+ {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
Available Stats Collectors
==========================
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index bda72f5e6..fa5e8d5c2 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -246,11 +246,14 @@ class ScrapyAgent(object):
expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
if maxsize and expected_size > maxsize:
- logger.error("Expected response size (%(size)s) larger than "
- "download max size (%(maxsize)s).",
- {'size': expected_size, 'maxsize': maxsize})
+ error_message = ("Cancelling download of {url}: expected response "
+ "size ({size}) larger than "
+ "download max size ({maxsize})."
+ ).format(url=request.url, size=expected_size, maxsize=maxsize)
+
+ logger.error(error_message)
txresponse._transport._producer.loseConnection()
- raise defer.CancelledError()
+ raise defer.CancelledError(error_message)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 803ed5fc0..3324aa21a 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -56,7 +56,7 @@ class RetryMiddleware(object):
def process_exception(self, request, exception, spider):
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
and not request.meta.get('dont_retry', False):
- return self._retry(request, exception, spider)
+ return self._retry(request, exception, spider)
def _retry(self, request, reason, spider):
retries = request.meta.get('retry_times', 0) + 1
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index ce59c9719..b0688791e 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
def xmliter(obj, nodename):
"""Return a iterator of Selector's over all nodes of a XML document,
- given tha name of the node to iterate. Useful for parsing XML feeds.
+ given the name of the node to iterate. Useful for parsing XML feeds.
obj can be:
- a Response object
@@ -35,7 +35,7 @@ def xmliter(obj, nodename):
header_end = re_rsearch(HEADER_END_RE, text)
header_end = text[header_end[1]:].strip() if header_end else ''
- r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL)
+ r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL)
for match in r.finditer(text):
nodetext = header_start + match.group() + header_end
yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
@@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
for _, node in iterable:
- nodetext = etree.tostring(node)
+ nodetext = etree.tostring(node, encoding='unicode')
node.clear()
xs = Selector(text=nodetext, type='xml')
if namespace:
@@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
def _body_or_str(obj, unicode=True):
- assert isinstance(obj, (Response, six.string_types, bytes)), \
- "obj must be Response or basestring, not %s" % type(obj).__name__
+ expected_types = (Response, six.text_type, six.binary_type)
+ assert isinstance(obj, expected_types), \
+ "obj must be %s, not %s" % (
+ " or ".join(t.__name__ for t in expected_types),
+ type(obj).__name__)
if isinstance(obj, Response):
if not unicode:
return obj.body
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index d42ed2c91..b2e3889a4 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import os
import six
from twisted.trial import unittest
@@ -46,6 +47,60 @@ class XmliterTestCase(unittest.TestCase):
for e in self.xmliter(response, 'matchme...')]
self.assertEqual(nodenames, [['matchme...']])
+ def test_xmliter_unicode(self):
+ # example taken from https://github.com/scrapy/scrapy/issues/1665
+ body = u"""
+ <þingflokkar>
+ <þingflokkur id="26">
+
+
+ -
+
+
+
+ 80
+
+
+ <þingflokkur id="21">
+ Alþýðubandalag
+
+ Ab
+ Alþb.
+
+
+ 76
+ 123
+
+
+ <þingflokkur id="27">
+ Alþýðuflokkur
+
+ A
+ Alþfl.
+
+
+ 27
+ 120
+
+
+ """
+
+ for r in (
+ # with bytes
+ XmlResponse(url="http://example.com", body=body.encode('utf-8')),
+ # Unicode body needs encoding information
+ XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
+
+ attrs = []
+ for x in self.xmliter(r, u'þingflokkur'):
+ attrs.append((x.xpath('@id').extract(),
+ x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
+ x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
+
+ self.assertEqual(attrs,
+ [([u'26'], [u'-'], [u'80']),
+ ([u'21'], [u'Ab'], [u'76']),
+ ([u'27'], [u'A'], [u'27'])])
def test_xmliter_text(self):
body = u"""onetwo"""
@@ -96,6 +151,10 @@ class XmliterTestCase(unittest.TestCase):
self.assertRaises(StopIteration, next, iter)
+ def test_xmliter_objtype_exception(self):
+ i = self.xmliter(42, 'product')
+ self.assertRaises(AssertionError, next, i)
+
def test_xmliter_encoding(self):
body = b'\n\n - Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6
\n\n\n'
response = XmlResponse('http://www.example.com', body=body)
@@ -169,6 +228,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
node = next(my_iter)
self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
+ def test_xmliter_objtype_exception(self):
+ i = self.xmliter(42, 'product')
+ self.assertRaises(TypeError, next, i)
class UtilsCsvTestCase(unittest.TestCase):
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')