pipeline media removed multispider support

This commit is contained in:
nramirezuy 2013-07-03 18:08:15 -03:00
parent 06b9702984
commit 2abb9d08cb
2 changed files with 3 additions and 9 deletions

View File

@ -19,7 +19,6 @@ class MediaPipeline(object):
self.waiting = defaultdict(list)
def __init__(self, download_func=None):
self.spiderinfo = {}
self.download_func = download_func
@classmethod
@ -32,13 +31,10 @@ class MediaPipeline(object):
return pipe
def open_spider(self, spider):
self.spiderinfo[spider] = self.SpiderInfo(spider)
def close_spider(self, spider):
del self.spiderinfo[spider]
self.spiderinfo = self.SpiderInfo(spider)
def process_item(self, item, spider):
info = self.spiderinfo[spider]
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info) for r in requests]
dfd = DeferredList(dlist, consumeErrors=1)

View File

@ -8,7 +8,6 @@ from scrapy.http import Request, Response
from scrapy.spider import BaseSpider
from scrapy.utils.request import request_fingerprint
from scrapy.contrib.pipeline.media import MediaPipeline
from scrapy.utils.test import get_crawler
from scrapy.utils.signal import disconnect_all
from scrapy import signals
from scrapy import log
@ -27,13 +26,12 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
self.spider = BaseSpider('media.com')
self.pipe = self.pipeline_class(download_func=_mocked_download_func)
self.pipe.open_spider(self.spider)
self.info = self.pipe.spiderinfo[self.spider]
self.info = self.pipe.spiderinfo
def tearDown(self):
for name, signal in vars(signals).items():
if not name.startswith('_'):
disconnect_all(signal)
self.pipe.close_spider(self.spider)
def test_default_media_to_download(self):
request = Request('http://url')