From 6f264ab190882d9bfb375688a7e44d03716242ba Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 30 Jan 2018 05:47:28 +0500 Subject: [PATCH 1/5] more stats for RobotsTxtMiddleware --- scrapy/downloadermiddlewares/robotstxt.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index c3dfa7819..b86c09c14 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -41,10 +41,12 @@ class RobotsTxtMiddleware(object): return d def process_request_2(self, rp, request, spider): - if rp is not None and not rp.can_fetch( - to_native_str(self._useragent), request.url): + if rp is None: + return + if not rp.can_fetch(to_native_str(self._useragent), request.url): logger.debug("Forbidden by robots.txt: %(request)s", {'request': request}, extra={'spider': spider}) + self.crawler.stats.inc_value('robotstxt/forbidden') raise IgnoreRequest() def robot_parser(self, request, spider): @@ -63,6 +65,7 @@ class RobotsTxtMiddleware(object): dfd.addCallback(self._parse_robots, netloc) dfd.addErrback(self._logerror, robotsreq, spider) dfd.addErrback(self._robots_error, netloc) + self.crawler.stats.inc_value('robotstxt/request_count') if isinstance(self._parsers[netloc], Deferred): d = Deferred() @@ -83,11 +86,14 @@ class RobotsTxtMiddleware(object): return failure def _parse_robots(self, response, netloc): + self.crawler.stats.inc_value('robotstxt/response_count') + self.crawler.stats.inc_value( + 'robotstxt/response_status_count/{}'.format(response.status)) rp = robotparser.RobotFileParser(response.url) body = '' if hasattr(response, 'text'): body = response.text - else: # last effort try + else: # last effort try try: body = response.body.decode('utf-8') except UnicodeDecodeError: @@ -95,7 +101,7 @@ class RobotsTxtMiddleware(object): # but keep the lookup cached (in self._parsers) # Running rp.parse() will set rp state from # 'disallow all' to 'allow any'. - pass + self.crawler.stats.inc_value('robotstxt/unicode_error_count') # stdlib's robotparser expects native 'str' ; # with unicode input, non-ASCII encoded bytes decoding fails in Python2 rp.parse(to_native_str(body).splitlines()) @@ -105,6 +111,9 @@ class RobotsTxtMiddleware(object): rp_dfd.callback(rp) def _robots_error(self, failure, netloc): + if failure.type is not IgnoreRequest: + key = 'robotstxt/exception_count/{}'.format(failure.type) + self.crawler.stats.inc_value(key) rp_dfd = self._parsers[netloc] self._parsers[netloc] = None rp_dfd.callback(None) From a56540877c2c24d3ba787cc43ca1f81d91b386fd Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 16 Jan 2018 16:14:35 -0300 Subject: [PATCH 2/5] Do not serialize unpickable objects (py3) --- scrapy/squeues.py | 7 ++++--- tests/test_squeues.py | 9 +++++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 21520f454..0b8f6af7d 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python>=3.5 raises AttributeError here while - # Python<=3.4 raises pickle.PicklingError - except (pickle.PicklingError, AttributeError) as e: + # Python<=3.4 raises pickle.PicklingError here while + # Python>=3.5 raises AttributeError and + # Python>=3.6 raises TypeError + except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ diff --git a/tests/test_squeues.py b/tests/test_squeues.py index 3a24348b4..d2f721241 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,3 +1,4 @@ +from sys import version_info import pickle from queuelib.tests import test_queue as t @@ -5,6 +6,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif from scrapy.item import Item, Field from scrapy.http import Request from scrapy.loader import ItemLoader +from scrapy.selector import Selector class TestItem(Item): name = Field() @@ -17,20 +19,23 @@ class TestLoader(ItemLoader): name_out = staticmethod(_test_procesor) def nonserializable_object_test(self): + q = self.queue() try: pickle.dumps(lambda x: x) except Exception: # Trigger Twisted bug #7989 import twisted.persisted.styles # NOQA - q = self.queue() self.assertRaises(ValueError, q.push, lambda x: x) else: # Use a different unpickleable object class A(object): pass a = A() a.__reduce__ = a.__reduce_ex__ = None - q = self.queue() self.assertRaises(ValueError, q.push, a) + if version_info.major == 3 and version_info.minor >= 6: + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From e4558cb27e8eeec8432a06124acf8c2569784ccc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 19 Jan 2018 10:51:30 -0300 Subject: [PATCH 3/5] Update test for unpickable objects --- tests/test_squeues.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/test_squeues.py b/tests/test_squeues.py index d2f721241..3ded5c027 100644 --- a/tests/test_squeues.py +++ b/tests/test_squeues.py @@ -1,4 +1,3 @@ -from sys import version_info import pickle from queuelib.tests import test_queue as t @@ -32,10 +31,9 @@ def nonserializable_object_test(self): a = A() a.__reduce__ = a.__reduce_ex__ = None self.assertRaises(ValueError, q.push, a) - if version_info.major == 3 and version_info.minor >= 6: - # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) - sel = Selector(text='

some text

') - self.assertRaises(ValueError, q.push, sel) + # Selectors should fail (lxml.html.HtmlElement objects can't be pickled) + sel = Selector(text='

some text

') + self.assertRaises(ValueError, q.push, sel) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From 0d87e77afeb506c69f4717744917b95234a86650 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Feb 2018 14:49:26 -0300 Subject: [PATCH 4/5] Bump parsel dependency --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 392f83dd6..2a94d742d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ queuelib six>=1.5.2 PyDispatcher>=2.0.5 service_identity -parsel>=1.1 +parsel>=1.4 diff --git a/setup.py b/setup.py index 06a36e2ba..c37919cda 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setup( 'pyOpenSSL', 'cssselect>=0.9', 'six>=1.5.2', - 'parsel>=1.1', + 'parsel>=1.4', 'PyDispatcher>=2.0.5', 'service_identity', ], From 6edd4114c4e715a3a0c440af455fff089a099620 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 8 Feb 2018 15:47:20 -0300 Subject: [PATCH 5/5] Clarify comment about Pyhton versions --- scrapy/squeues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 0b8f6af7d..d2074a457 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -25,9 +25,9 @@ def _serializable_queue(queue_class, serialize, deserialize): def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) - # Python<=3.4 raises pickle.PicklingError here while - # Python>=3.5 raises AttributeError and - # Python>=3.6 raises TypeError + # Python <= 3.4 raises pickle.PicklingError here while + # 3.5 <= Python < 3.6 raises AttributeError and + # Python >= 3.6 raises TypeError except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e))