From 016875fd513c66b7ee1bcc80a1f2ae0d6ffda2d2 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 15:30:06 +0300 Subject: [PATCH 01/11] added more verbosity for log and for exception when download is cancelled because of a size limit --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..78f3f74fa 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,11 +239,16 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - logger.error("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s).", - {'size': expected_size, 'maxsize': maxsize}) + error_message = ( + "Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize}).".format( + url=request.url, size=expected_size, maxsize=maxsize + ) + ) + logger.error(error_message) txresponse._transport._producer.loseConnection() - raise defer.CancelledError() + raise defer.CancelledError(error_message) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From 37289815ac98404458c43159abfd46dd54ed4b7b Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 16:15:50 +0300 Subject: [PATCH 02/11] code formatting fix --- scrapy/core/downloader/handlers/http11.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 78f3f74fa..9d075e877 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,13 +239,11 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ( - "Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize}).".format( - url=request.url, size=expected_size, maxsize=maxsize - ) - ) + error_message = ("Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize})." + ).format(url=request.url, size=expected_size, maxsize=maxsize) + logger.error(error_message) txresponse._transport._producer.loseConnection() raise defer.CancelledError(error_message) From d03d262f5290d7a439ac797947413894f158f425 Mon Sep 17 00:00:00 2001 From: palego Date: Mon, 4 Jan 2016 10:00:13 +0100 Subject: [PATCH 03/11] indentation --- scrapy/downloadermiddlewares/retry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 803ed5fc0..3324aa21a 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -56,7 +56,7 @@ class RetryMiddleware(object): def process_exception(self, request, exception, spider): if isinstance(exception, self.EXCEPTIONS_TO_RETRY) \ and not request.meta.get('dont_retry', False): - return self._retry(request, exception, spider) + return self._retry(request, exception, spider) def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 From 6ddd8147382348f4796ac905f0357925dfa81da1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 12 Jan 2016 10:48:45 +0100 Subject: [PATCH 04/11] Support unicode tags in xml iterators (fixes #1665) --- scrapy/utils/iterators.py | 10 +++---- tests/test_utils_iterators.py | 51 ++++++++++++++++++++++++++++++++++- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c0d93f7a9..bec985062 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -18,7 +18,7 @@ logger = logging.getLogger(__name__) def xmliter(obj, nodename): """Return a iterator of Selector's over all nodes of a XML document, - given tha name of the node to iterate. Useful for parsing XML feeds. + given the name of the node to iterate. Useful for parsing XML feeds. obj can be: - a Response object @@ -36,7 +36,7 @@ def xmliter(obj, nodename): header_end = re_rsearch(HEADER_END_RE, text) header_end = text[header_end[1]:].strip() if header_end else '' - r = re.compile(r"<{0}[\s>].*?".format(nodename_patt), re.DOTALL) + r = re.compile(r'<%(np)s[\s>].*?' % {'np': nodename_patt}, re.DOTALL) for match in r.finditer(text): nodetext = header_start + match.group() + header_end yield Selector(text=nodetext, type='xml').xpath('//' + nodename)[0] @@ -49,7 +49,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node) + nodetext = etree.tostring(node, encoding='unicode') node.clear() xs = Selector(text=nodetext, type='xml') if namespace: @@ -94,7 +94,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): headers is an iterable that when provided offers the keys for the returned dictionaries, if not the first row is used. - + quotechar is the character used to enclosure fields on the given obj. """ @@ -125,7 +125,7 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types)), \ + assert isinstance(obj, (Response, six.string_types, bytes)), \ "obj must be Response or basestring, not %s" % type(obj).__name__ if isinstance(obj, Response): if not unicode: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 590c53302..8dceed7cc 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- import os from twisted.trial import unittest @@ -45,6 +46,54 @@ class XmliterTestCase(unittest.TestCase): for e in self.xmliter(response, 'matchme...')] self.assertEqual(nodenames, [['matchme...']]) + def test_xmliter_unicode(self): + # example taken from https://github.com/scrapy/scrapy/issues/1665 + body = """ + <þingflokkar> + <þingflokkur id="26"> + + + - + + + + 80 + + + <þingflokkur id="21"> + Alþýðubandalag + + Ab + Alþb. + + + 76 + 123 + + + <þingflokkur id="27"> + Alþýðuflokkur + + A + Alþfl. + + + 27 + 120 + + + """ + response = XmlResponse(url="http://example.com", body=body) + attrs = [] + for x in self.xmliter(response, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" @@ -206,7 +255,7 @@ class UtilsCsvTestCase(unittest.TestCase): def test_csviter_quotechar(self): body1 = get_testdata('feeds', 'feed-sample6.csv') body2 = get_testdata('feeds', 'feed-sample6.csv').replace(",", '|') - + response1 = TextResponse(url="http://example.com/", body=body1) csv1 = csviter(response1, quotechar="'") From 9fad25f3d14091d250cc4b1d668befca00c30ef0 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 11:42:41 +0100 Subject: [PATCH 05/11] Use explicit Unicode and bytes for XML body in tests --- scrapy/utils/iterators.py | 9 ++++++--- tests/test_utils_iterators.py | 19 +++++++++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index c215a0bdd..69c7f2c23 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding='unicode') + nodetext = etree.tostring(node, encoding=six.text_type) node.clear() xs = Selector(text=nodetext, type='xml') if namespace: @@ -128,8 +128,11 @@ def csviter(obj, delimiter=None, headers=None, encoding=None, quotechar=None): def _body_or_str(obj, unicode=True): - assert isinstance(obj, (Response, six.string_types, bytes)), \ - "obj must be Response or basestring, not %s" % type(obj).__name__ + expected_types = (Response, six.text_type, six.binary_type) + assert isinstance(obj, expected_types), \ + "obj must be %s, not %s" % ( + " or ".join(t.__name__ for t in expected_types), + type(obj).__name__) if isinstance(obj, Response): if not unicode: return obj.body diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index de103fea5..74c22d420 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -49,7 +49,7 @@ class XmliterTestCase(unittest.TestCase): def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 - body = """ + body = u""" <þingflokkar> <þingflokkur id="26"> @@ -84,7 +84,22 @@ class XmliterTestCase(unittest.TestCase): """ - response = XmlResponse(url="http://example.com", body=body) + + # with bytes + response = XmlResponse(url="http://example.com", body=body.encode('utf-8')) + attrs = [] + for x in self.xmliter(response, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) + + # Unicode body needs encoding information + response = XmlResponse(url="http://example.com", body=body, encoding='utf-8') attrs = [] for x in self.xmliter(response, u'þingflokkur'): attrs.append((x.xpath('@id').extract(), From d4c7d72b2b6fc20c1df0f697dd60801b93848628 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:13:47 +0100 Subject: [PATCH 06/11] Add tests for input type in xmliter calls --- tests/test_utils_iterators.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 74c22d420..8c4d6cf59 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -160,6 +160,10 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + def test_xmliter_objtype_exception(self): + i = self.xmliter(42, 'product') + self.assertRaises(AssertionError, next, i) + def test_xmliter_encoding(self): body = b'\n\n Some Turkish Characters \xd6\xc7\xde\xdd\xd0\xdc \xfc\xf0\xfd\xfe\xe7\xf6\n\n\n' response = XmlResponse('http://www.example.com', body=body) @@ -233,6 +237,9 @@ class LxmlXmliterTestCase(XmliterTestCase): node = next(my_iter) self.assertEqual(node.xpath('f:name/text()').extract(), ['African Coffee Table']) + def test_xmliter_objtype_exception(self): + i = self.xmliter(42, 'product') + self.assertRaises(TypeError, next, i) class UtilsCsvTestCase(unittest.TestCase): sample_feeds_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'feeds') From 1347015a80b9d8dafe0e1ac067d65b7e0e7c3f84 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 13 Jan 2016 12:32:28 +0100 Subject: [PATCH 07/11] Refactored test code --- tests/test_utils_iterators.py | 37 +++++++++++++---------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 8c4d6cf59..b2e3889a4 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -85,31 +85,22 @@ class XmliterTestCase(unittest.TestCase): """ - # with bytes - response = XmlResponse(url="http://example.com", body=body.encode('utf-8')) - attrs = [] - for x in self.xmliter(response, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) + for r in ( + # with bytes + XmlResponse(url="http://example.com", body=body.encode('utf-8')), + # Unicode body needs encoding information + XmlResponse(url="http://example.com", body=body, encoding='utf-8')): - self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + attrs = [] + for x in self.xmliter(r, u'þingflokkur'): + attrs.append((x.xpath('@id').extract(), + x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), + x.xpath(u'./tímabil/fyrstaþing/text()').extract())) - # Unicode body needs encoding information - response = XmlResponse(url="http://example.com", body=body, encoding='utf-8') - attrs = [] - for x in self.xmliter(response, u'þingflokkur'): - attrs.append((x.xpath('@id').extract(), - x.xpath(u'./skammstafanir/stuttskammstöfun/text()').extract(), - x.xpath(u'./tímabil/fyrstaþing/text()').extract())) - - self.assertEqual(attrs, - [([u'26'], [u'-'], [u'80']), - ([u'21'], [u'Ab'], [u'76']), - ([u'27'], [u'A'], [u'27'])]) + self.assertEqual(attrs, + [([u'26'], [u'-'], [u'80']), + ([u'21'], [u'Ab'], [u'76']), + ([u'27'], [u'A'], [u'27'])]) def test_xmliter_text(self): body = u"""onetwo""" From bb38400db560a4d814ea6c12364404d6daa75da5 Mon Sep 17 00:00:00 2001 From: Ralph Gutkowski Date: Fri, 15 Jan 2016 19:00:58 +0100 Subject: [PATCH 08/11] Update Stats Collection documentation `pages_crawled` value doesn't exist. Replace it with `downloader/response_count` in order to avoid confusion. --- docs/topics/stats.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 0837610d0..290fc065c 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -47,7 +47,7 @@ Set stat value:: Increment stat value:: - stats.inc_value('pages_crawled') + stats.inc_value('downloader/response_count') Set stat value only if greater than previous:: @@ -59,13 +59,13 @@ Set stat value only if lower than previous:: Get stat value:: - >>> stats.get_value('pages_crawled') + >>> stats.get_value('downloader/response_count') 8 Get all stats:: >>> stats.get_stats() - {'pages_crawled': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 79147a61a797f81fee44b000eb5ac9591593ff88 Mon Sep 17 00:00:00 2001 From: Ralph Gutkowski Date: Fri, 15 Jan 2016 19:25:56 +0100 Subject: [PATCH 09/11] Update stats.rst --- docs/topics/stats.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 290fc065c..dd0c6216b 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -47,7 +47,7 @@ Set stat value:: Increment stat value:: - stats.inc_value('downloader/response_count') + stats.inc_value('custom_count') Set stat value only if greater than previous:: @@ -59,13 +59,13 @@ Set stat value only if lower than previous:: Get stat value:: - >>> stats.get_value('downloader/response_count') - 8 + >>> stats.get_value('custom_count') + 1 Get all stats:: >>> stats.get_stats() - {'downloader/response_count': 1238, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} + {'custom_count': 1, 'start_time': datetime.datetime(2009, 7, 14, 21, 47, 28, 977139)} Available Stats Collectors ========================== From 4e44766653c294c54a3bb960b3734ee70bc663a4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 15 Jan 2016 19:51:21 +0100 Subject: [PATCH 10/11] Use "unicode" string for lxml.etree.tostring() serialization --- scrapy/utils/iterators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 69c7f2c23..b0688791e 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -48,7 +48,7 @@ def xmliter_lxml(obj, nodename, namespace=None, prefix='x'): iterable = etree.iterparse(reader, tag=tag, encoding=reader.encoding) selxpath = '//' + ('%s:%s' % (prefix, nodename) if namespace else nodename) for _, node in iterable: - nodetext = etree.tostring(node, encoding=six.text_type) + nodetext = etree.tostring(node, encoding='unicode') node.clear() xs = Selector(text=nodetext, type='xml') if namespace: From cd735e377c2a26bc8ebb925bf6031429549faadf Mon Sep 17 00:00:00 2001 From: cclauss Date: Mon, 18 Jan 2016 07:45:36 +0100 Subject: [PATCH 11/11] Simplify if statement --- conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index b0ac1badd..8b4faf8fc 100644 --- a/conftest.py +++ b/conftest.py @@ -34,7 +34,7 @@ if (twisted_version.major, twisted_version.minor, twisted_version.micro) >= (15, if six.PY3: for line in open('tests/py3-ignores.txt'): file_path = line.strip() - if len(file_path) > 0 and file_path[0] != '#': + if file_path and file_path[0] != '#': collect_ignore.append(file_path)