RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3476 from elacuesta/deque_appendleft 

[MRG+1] Use collections.deque instead of list to store MiddlewareManager's methods
This commit is contained in:
Daniel Graña 2018-12-26 11:13:52 -03:00 committed by GitHub
parent 76433feb4f f97e3e90f2
commit f6ce716372
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/core/downloader/middleware.py
View File

@ -26,9 +26,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_request'):
self.methods['process_request'].append(mw.process_request)
if hasattr(mw, 'process_response'):
self.methods['process_response'].insert(0, mw.process_response)
self.methods['process_response'].appendleft(mw.process_response)
if hasattr(mw, 'process_exception'):
self.methods['process_exception'].insert(0, mw.process_exception)
self.methods['process_exception'].appendleft(mw.process_exception)
def download(self, download_func, request, spider):
@defer.inlineCallbacks

6
scrapy/core/spidermw.py
View File

@ -25,11 +25,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
if hasattr(mw, 'process_spider_input'):
self.methods['process_spider_input'].append(mw.process_spider_input)
if hasattr(mw, 'process_spider_output'):
self.methods['process_spider_output'].insert(0, mw.process_spider_output)
self.methods['process_spider_output'].appendleft(mw.process_spider_output)
if hasattr(mw, 'process_spider_exception'):
self.methods['process_spider_exception'].insert(0, mw.process_spider_exception)
self.methods['process_spider_exception'].appendleft(mw.process_spider_exception)
if hasattr(mw, 'process_start_requests'):
self.methods['process_start_requests'].insert(0, mw.process_start_requests)
self.methods['process_start_requests'].appendleft(mw.process_start_requests)
def scrape_response(self, scrape_func, response, request, spider):
fname = lambda f:'%s.%s' % (

6
scrapy/middleware.py
View File

@ -1,4 +1,4 @@
from collections import defaultdict
from collections import defaultdict, deque
import logging
import pprint
@ -16,7 +16,7 @@ class MiddlewareManager(object):
def __init__(self, *middlewares):
self.middlewares = middlewares
self.methods = defaultdict(list)
self.methods = defaultdict(deque)
for mw in middlewares:
self._add_middleware(mw)
@ -56,7 +56,7 @@ class MiddlewareManager(object):
if hasattr(mw, 'open_spider'):
self.methods['open_spider'].append(mw.open_spider)
if hasattr(mw, 'close_spider'):
self.methods['close_spider'].insert(0, mw.close_spider)
self.methods['close_spider'].appendleft(mw.close_spider)
def _process_parallel(self, methodname, obj, *args):
return process_parallel(self.methods[methodname], obj, *args)

6
tests/test_middleware.py
View File

@ -60,9 +60,9 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_init(self):
m1, m2, m3 = M1(), M2(), M3()
mwman = TestMiddlewareManager(m1, m2, m3)
self.assertEqual(mwman.methods['open_spider'], [m1.open_spider, m2.open_spider])
self.assertEqual(mwman.methods['close_spider'], [m2.close_spider, m1.close_spider])
self.assertEqual(mwman.methods['process'], [m1.process, m3.process])
self.assertEqual(list(mwman.methods['open_spider']), [m1.open_spider, m2.open_spider])
self.assertEqual(list(mwman.methods['close_spider']), [m2.close_spider, m1.close_spider])
self.assertEqual(list(mwman.methods['process']), [m1.process, m3.process])
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())