fix error messages due to fetching pages during shutdown process

This version keeps the faster approach of not processing request callbacks when engine is shutting down

scrapy/core/scheduler.py

@@ -58,7 +58,7 @@ class Scheduler(object):
         return dfd
 
     def clear_pending_requests(self, spider):
-        pass
+        self.pending_requests[spider].clear()
 
     def next_request(self, spider):
         """Return the next available request to be downloaded for a spider.

scrapy/utils/datatypes.py

@@ -305,6 +305,11 @@ class PriorityQueue(object):
                         del self.positems[priority]
                     return t
         raise IndexError("pop from an empty queue")
+    
+    def clear(self):
+        self.negitems.clear()
+        self.pzero.clear()
+        self.positems.clear()
 
     def __len__(self):
         total = sum(len(v) for v in self.negitems.values()) + \