From f97e3e90f25c5077b47d9ec11a4cf84ea777227e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 29 Oct 2018 12:40:20 -0300 Subject: [PATCH] Use collections.deque instead of list to store methods --- scrapy/core/downloader/middleware.py | 4 ++-- scrapy/core/spidermw.py | 6 +++--- scrapy/middleware.py | 6 +++--- tests/test_middleware.py | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index c3b23e284..f5e2fca63 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_request'): self.methods['process_request'].append(mw.process_request) if hasattr(mw, 'process_response'): - self.methods['process_response'].insert(0, mw.process_response) + self.methods['process_response'].appendleft(mw.process_response) if hasattr(mw, 'process_exception'): - self.methods['process_exception'].insert(0, mw.process_exception) + self.methods['process_exception'].appendleft(mw.process_exception) def download(self, download_func, request, spider): @defer.inlineCallbacks diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a206e4b0c..16b8435ab 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager): if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_spider_output'): - self.methods['process_spider_output'].insert(0, mw.process_spider_output) + self.methods['process_spider_output'].appendleft(mw.process_spider_output) if hasattr(mw, 'process_spider_exception'): - self.methods['process_spider_exception'].insert(0, mw.process_spider_exception) + self.methods['process_spider_exception'].appendleft(mw.process_spider_exception) if hasattr(mw, 'process_start_requests'): - self.methods['process_start_requests'].insert(0, mw.process_start_requests) + self.methods['process_start_requests'].appendleft(mw.process_start_requests) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( diff --git a/scrapy/middleware.py b/scrapy/middleware.py index f2240984c..1cfd8a782 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -1,4 +1,4 @@ -from collections import defaultdict +from collections import defaultdict, deque import logging import pprint @@ -16,7 +16,7 @@ class MiddlewareManager(object): def __init__(self, *middlewares): self.middlewares = middlewares - self.methods = defaultdict(list) + self.methods = defaultdict(deque) for mw in middlewares: self._add_middleware(mw) @@ -56,7 +56,7 @@ class MiddlewareManager(object): if hasattr(mw, 'open_spider'): self.methods['open_spider'].append(mw.open_spider) if hasattr(mw, 'close_spider'): - self.methods['close_spider'].insert(0, mw.close_spider) + self.methods['close_spider'].appendleft(mw.close_spider) def _process_parallel(self, methodname, obj, *args): return process_parallel(self.methods[methodname], obj, *args) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6d885330..aea0be825 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase): def test_init(self): m1, m2, m3 = M1(), M2(), M3() mwman = TestMiddlewareManager(m1, m2, m3) - self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider]) - self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider]) - self.assertEqual(mwman.methods['process'], [m1.process, m3.process]) + self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider]) + self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider]) + self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process]) def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3())