mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into py3-http-downloaders
This commit is contained in:
Konstantin Lopuhin
commit
07577c432e
|
|
@ -34,7 +34,7 @@ if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15,
|
|||
if six.PY3:
|
||||
for line in open('tests/py3-ignores.txt'):
|
||||
file_path = line.strip()
|
||||
if len(file_path) > 0 and file_path[0] != '#':
|
||||
if file_path and file_path[0] != '#':
|
||||
collect_ignore.append(file_path)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ Set stat value::
|
|||
|
||||
Increment stat value::
|
||||
|
||||
stats.inc_value('pages_crawled')
|
||||
stats.inc_value('custom_count')
|
||||
|
||||
Set stat value only if greater than previous::
|
||||
|
||||
|
|
@ -59,13 +59,13 @@ Set stat value only if lower than previous::
|
|||
|
||||
Get stat value::
|
||||
|
||||
>>> stats.get_value('pages_crawled')
|
||||
8
|
||||
>>> stats.get_value('custom_count')
|
||||
1
|
||||
|
||||
Get all stats::
|
||||
|
||||
>>> stats.get_stats()
|
||||
{'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
|
||||
{'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)}
|
||||
|
||||
Available Stats Collectors
|
||||
==========================
|
||||
|
|
|
|||
|
|
@ -246,11 +246,14 @@ class ScrapyAgent(object):
|
|||
expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1
|
||||
|
||||
if maxsize and expected_size > maxsize:
|
||||
logger.error("Expected response size (%(size)s) larger than "
|
||||
"download max size (%(maxsize)s).",
|
||||
{'size': expected_size, 'maxsize': maxsize})
|
||||
error_message = ("Cancelling download of {url}: expected response "
|
||||
"size ({size}) larger than "
|
||||
"download max size ({maxsize})."
|
||||
).format(url=request.url, size=expected_size, maxsize=maxsize)
|
||||
|
||||
logger.error(error_message)
|
||||
txresponse._transport._producer.loseConnection()
|
||||
raise defer.CancelledError()
|
||||
raise defer.CancelledError(error_message)
|
||||
|
||||
if warnsize and expected_size > warnsize:
|
||||
logger.warning("Expected response size (%(size)s) larger than "
|
||||
|
|
|
|||
|
|
@ -56,7 +56,7 @@ class RetryMiddleware(object):
|
|||
def process_exception(self, request, exception, spider):
|
||||
if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \
|
||||
and not request.meta.get('dont_retry', False):
|
||||
return self._retry(request, exception, spider)
|
||||
return self._retry(request, exception, spider)
|
||||
|
||||
def _retry(self, request, reason, spider):
|
||||
retries = request.meta.get('retry_times', 0) + 1
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
def xmliter(obj, nodename):
|
||||
"""Return a iterator of Selector's over all nodes of a XML document,
|
||||
given tha name of the node to iterate. Useful for parsing XML feeds.
|
||||
given the name of the node to iterate. Useful for parsing XML feeds.
|
||||
|
||||
obj can be:
|
||||
- a Response object
|
||||
|
|
@ -35,7 +35,7 @@ def xmliter(obj, nodename):
|
|||
header_end = re_rsearch(HEADER_END_RE, text)
|
||||
header_end = text[header_end[1]:].strip() if header_end else ''
|
||||
|
||||
r = re.compile(r"<{0}[\s>].*?</{0}>".format(nodename_patt), re.DOTALL)
|
||||
r = re.compile(r'<%(np)s[\s>].*?</%(np)s>' % {'np': nodename_patt}, re.DOTALL)
|
||||
for match in r.finditer(text):
|
||||
nodetext = header_start + match.group() + header_end
|
||||
yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0]
|
||||
|
|
@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'):
|
|||
iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding)
|
||||
selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename)
|
||||
for _, node in iterable:
|
||||
nodetext = etree.tostring(node)
|
||||
nodetext = etree.tostring(node, encoding='unicode')
|
||||
node.clear()
|
||||
xs = Selector(text=nodetext, type='xml')
|
||||
if namespace:
|
||||
|
|
@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None):
|
|||
|
||||
|
||||
def _body_or_str(obj, unicode=True):
|
||||
assert isinstance(obj, (Response, six.string_types, bytes)), \
|
||||
"obj must be Response or basestring, not %s" % type(obj).__name__
|
||||
expected_types = (Response, six.text_type, six.binary_type)
|
||||
assert isinstance(obj, expected_types), \
|
||||
"obj must be %s, not %s" % (
|
||||
" or ".join(t.__name__ for t in expected_types),
|
||||
type(obj).__name__)
|
||||
if isinstance(obj, Response):
|
||||
if not unicode:
|
||||
return obj.body
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import six
|
||||
from twisted.trial import unittest
|
||||
|
|
@ -46,6 +47,60 @@ class XmliterTestCase(unittest.TestCase):
|
|||
for e in self.xmliter(response, 'matchme...')]
|
||||
self.assertEqual(nodenames, [['matchme...']])
|
||||
|
||||
def test_xmliter_unicode(self):
|
||||
# example taken from https://github.com/scrapy/scrapy/issues/1665
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?>
|
||||
<þingflokkar>
|
||||
<þingflokkur id="26">
|
||||
<heiti />
|
||||
<skammstafanir>
|
||||
<stuttskammstöfun>-</stuttskammstöfun>
|
||||
<löngskammstöfun />
|
||||
</skammstafanir>
|
||||
<tímabil>
|
||||
<fyrstaþing>80</fyrstaþing>
|
||||
</tímabil>
|
||||
</þingflokkur>
|
||||
<þingflokkur id="21">
|
||||
<heiti>Alþýðubandalag</heiti>
|
||||
<skammstafanir>
|
||||
<stuttskammstöfun>Ab</stuttskammstöfun>
|
||||
<löngskammstöfun>Alþb.</löngskammstöfun>
|
||||
</skammstafanir>
|
||||
<tímabil>
|
||||
<fyrstaþing>76</fyrstaþing>
|
||||
<síðastaþing>123</síðastaþing>
|
||||
</tímabil>
|
||||
</þingflokkur>
|
||||
<þingflokkur id="27">
|
||||
<heiti>Alþýðuflokkur</heiti>
|
||||
<skammstafanir>
|
||||
<stuttskammstöfun>A</stuttskammstöfun>
|
||||
<löngskammstöfun>Alþfl.</löngskammstöfun>
|
||||
</skammstafanir>
|
||||
<tímabil>
|
||||
<fyrstaþing>27</fyrstaþing>
|
||||
<síðastaþing>120</síðastaþing>
|
||||
</tímabil>
|
||||
</þingflokkur>
|
||||
</þingflokkar>"""
|
||||
|
||||
for r in (
|
||||
# with bytes
|
||||
XmlResponse(url="http://example.com", body=body.encode('utf-8')),
|
||||
# Unicode body needs encoding information
|
||||
XmlResponse(url="http://example.com", body=body, encoding='utf-8')):
|
||||
|
||||
attrs = []
|
||||
for x in self.xmliter(r, u'þingflokkur'):
|
||||
attrs.append((x.xpath('@id').extract(),
|
||||
x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(),
|
||||
x.xpath(u'./tímabil/fyrstaþing/text()').extract()))
|
||||
|
||||
self.assertEqual(attrs,
|
||||
[([u'26'], [u'-'], [u'80']),
|
||||
([u'21'], [u'Ab'], [u'76']),
|
||||
([u'27'], [u'A'], [u'27'])])
|
||||
|
||||
def test_xmliter_text(self):
|
||||
body = u"""<?xml version="1.0" encoding="UTF-8"?><products><product>one</product><product>two</product></products>"""
|
||||
|
|
@ -96,6 +151,10 @@ class XmliterTestCase(unittest.TestCase):
|
|||
|
||||
self.assertRaises(StopIteration, next, iter)
|
||||
|
||||
def test_xmliter_objtype_exception(self):
|
||||
i = self.xmliter(42, 'product')
|
||||
self.assertRaises(AssertionError, next, i)
|
||||
|
||||
def test_xmliter_encoding(self):
|
||||
body = b'<?xml version="1.0" encoding="ISO-8859-9"?>\n<xml>\n <item>Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6</item>\n</xml>\n\n'
|
||||
response = XmlResponse('http://www.example.com', body=body)
|
||||
|
|
@ -169,6 +228,9 @@ class LxmlXmliterTestCase(XmliterTestCase):
|
|||
node = next(my_iter)
|
||||
self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table'])
|
||||
|
||||
def test_xmliter_objtype_exception(self):
|
||||
i = self.xmliter(42, 'product')
|
||||
self.assertRaises(TypeError, next, i)
|
||||
|
||||
class UtilsCsvTestCase(unittest.TestCase):
|
||||
sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds')
|
||||
|
|
|
|||
Loading…
Reference in New Issue