Merge branch 'master' into master

requirements.txt

@@ -7,4 +7,4 @@ queuelib
 six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
-parsel>=1.1
+parsel>=1.4

scrapy/downloadermiddlewares/robotstxt.py

@@ -41,10 +41,12 @@ class RobotsTxtMiddleware(object):
         return d
 
     def process_request_2(self, rp, request, spider):
-        if rp is not None and not rp.can_fetch(
-                 to_native_str(self._useragent), request.url):
+        if rp is None:
+            return
+        if not rp.can_fetch(to_native_str(self._useragent), request.url):
             logger.debug("Forbidden by robots.txt: %(request)s",
                          {'request': request}, extra={'spider': spider})
+            self.crawler.stats.inc_value('robotstxt/forbidden')
             raise IgnoreRequest("Forbidden by robots.txt")
 
     def robot_parser(self, request, spider):
@@ -63,6 +65,7 @@ class RobotsTxtMiddleware(object):
             dfd.addCallback(self._parse_robots, netloc)
             dfd.addErrback(self._logerror, robotsreq, spider)
             dfd.addErrback(self._robots_error, netloc)
+            self.crawler.stats.inc_value('robotstxt/request_count')
 
         if isinstance(self._parsers[netloc], Deferred):
             d = Deferred()
@@ -83,11 +86,14 @@ class RobotsTxtMiddleware(object):
         return failure
 
     def _parse_robots(self, response, netloc):
+        self.crawler.stats.inc_value('robotstxt/response_count')
+        self.crawler.stats.inc_value(
+            'robotstxt/response_status_count/{}'.format(response.status))
         rp = robotparser.RobotFileParser(response.url)
         body = ''
         if hasattr(response, 'text'):
             body = response.text
-        else: # last effort try
+        else:  # last effort try
             try:
                 body = response.body.decode('utf-8')
             except UnicodeDecodeError:
@@ -95,7 +101,7 @@ class RobotsTxtMiddleware(object):
                 # but keep the lookup cached (in self._parsers)
                 # Running rp.parse() will set rp state from
                 # 'disallow all' to 'allow any'.
-                pass
+                self.crawler.stats.inc_value('robotstxt/unicode_error_count')
         # stdlib's robotparser expects native 'str' ;
         # with unicode input, non-ASCII encoded bytes decoding fails in Python2
         rp.parse(to_native_str(body).splitlines())
@@ -105,6 +111,9 @@ class RobotsTxtMiddleware(object):
         rp_dfd.callback(rp)
 
     def _robots_error(self, failure, netloc):
+        if failure.type is not IgnoreRequest:
+            key = 'robotstxt/exception_count/{}'.format(failure.type)
+            self.crawler.stats.inc_value(key)
         rp_dfd = self._parsers[netloc]
         self._parsers[netloc] = None
         rp_dfd.callback(None)

scrapy/squeues.py

@@ -25,9 +25,10 @@ def _serializable_queue(queue_class, serialize, deserialize):
 def _pickle_serialize(obj):
     try:
         return pickle.dumps(obj, protocol=2)
-    # Python>=3.5 raises AttributeError here while
-    # Python<=3.4 raises pickle.PicklingError
-    except (pickle.PicklingError, AttributeError) as e:
+    # Python <= 3.4 raises pickle.PicklingError here while
+    # 3.5 <= Python < 3.6 raises AttributeError and
+    # Python >= 3.6 raises TypeError
+    except (pickle.PicklingError, AttributeError, TypeError) as e:
         raise ValueError(str(e))
 
 PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \

setup.py

@@ -71,7 +71,7 @@ setup(
         'pyOpenSSL',
         'cssselect>=0.9',
         'six>=1.5.2',
-        'parsel>=1.1',
+        'parsel>=1.4',
         'PyDispatcher>=2.0.5',
         'service_identity',
     ],

tests/test_squeues.py

@@ -5,6 +5,7 @@ from scrapy.squeues import MarshalFifoDiskQueue, MarshalLifoDiskQueue, PickleFif
 from scrapy.item import Item, Field
 from scrapy.http import Request
 from scrapy.loader import ItemLoader
+from scrapy.selector import Selector
 
 class TestItem(Item):
     name = Field()
@@ -17,20 +18,22 @@ class TestLoader(ItemLoader):
     name_out = staticmethod(_test_procesor)
 
 def nonserializable_object_test(self):
+    q = self.queue()
     try:
         pickle.dumps(lambda x: x)
     except Exception:
         # Trigger Twisted bug #7989
         import twisted.persisted.styles  # NOQA
-        q = self.queue()
         self.assertRaises(ValueError, q.push, lambda x: x)
     else:
         # Use a different unpickleable object
         class A(object): pass
         a = A()
         a.__reduce__ = a.__reduce_ex__ = None
-        q = self.queue()
         self.assertRaises(ValueError, q.push, a)
+    # Selectors should fail (lxml.html.HtmlElement objects can't be pickled)
+    sel = Selector(text='<html><body><p>some text</p></body></html>')
+    self.assertRaises(ValueError, q.push, sel)
 
 class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):