diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 26785bad5..022442783 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -24,7 +24,6 @@ from scrapy.http import Request from scrapy import signals from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest from scrapy.contrib.pipeline.media import MediaPipeline -from scrapy.conf import settings class NoimagesDrop(DropItem): @@ -78,8 +77,8 @@ class FSImagesStore(object): class S3ImagesStore(object): - AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] - AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + AWS_ACCESS_KEY_ID = None + AWS_SECRET_ACCESS_KEY = None def __init__(self, uri): assert uri.startswith('s3://') @@ -139,22 +138,33 @@ class ImagesPipeline(MediaPipeline): """ MEDIA_NAME = 'image' - MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0) - MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0) - EXPIRES = settings.getint('IMAGES_EXPIRES', 90) - THUMBS = settings.get('IMAGES_THUMBS', {}) + MIN_WIDTH = 0 + MIN_HEIGHT = 0 + EXPIRES = 90 + THUMBS = {} STORE_SCHEMES = { '': FSImagesStore, 'file': FSImagesStore, 's3': S3ImagesStore, } - def __init__(self): + def __init__(self, store_uri): + self.store = self._get_store(store_uri) + super(ImagesPipeline, self).__init__() + + @classmethod + def from_settings(cls, settings): + cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0) + cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0) + cls.EXPIRES = settings.getint('IMAGES_EXPIRES', 90) + cls.THUMBS = settings.get('IMAGES_THUMBS', {}) + s3store = cls.STORE_SCHEMES['s3'] + s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] + s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] store_uri = settings['IMAGES_STORE'] if not store_uri: raise NotConfigured - self.store = self._get_store(store_uri) - super(ImagesPipeline, self).__init__() + return cls(store_uri) def _get_store(self, uri): if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir diff --git a/scrapy/contrib/spidermiddleware/depth.py b/scrapy/contrib/spidermiddleware/depth.py index 4b15285d7..786dd9797 100644 --- a/scrapy/contrib/spidermiddleware/depth.py +++ b/scrapy/contrib/spidermiddleware/depth.py @@ -6,16 +6,24 @@ See documentation in docs/topics/spider-middleware.rst from scrapy import log from scrapy.http import Request -from scrapy.stats import stats -from scrapy.conf import settings class DepthMiddleware(object): - def __init__(self): - self.maxdepth = settings.getint('DEPTH_LIMIT') - self.stats = settings.getbool('DEPTH_STATS') + def __init__(self, maxdepth, stats=None): + self.maxdepth = maxdepth + self.stats = stats if self.stats and self.maxdepth: - stats.set_value('envinfo/request_depth_limit', self.maxdepth) + stats.set_value('envinfo/request_depth_limit', maxdepth) + + @classmethod + def from_settings(cls, settings): + maxdepth = settings.getint('DEPTH_LIMIT') + usestats = settings.getbool('DEPTH_STATS') + if usestats: + from scrapy.stats import stats + else: + stats = None + return cls(maxdepth, stats) def process_spider_output(self, response, result, spider): def _filter(request): @@ -27,14 +35,14 @@ class DepthMiddleware(object): level=log.DEBUG, spider=spider) return False elif self.stats: - stats.inc_value('request_depth_count/%s' % depth, spider=spider) - if depth > stats.get_value('request_depth_max', 0, spider=spider): - stats.set_value('request_depth_max', depth, spider=spider) + self.stats.inc_value('request_depth_count/%s' % depth, spider=spider) + if depth > self.stats.get_value('request_depth_max', 0, spider=spider): + self.stats.set_value('request_depth_max', depth, spider=spider) return True # base case (depth=0) if self.stats and 'depth' not in response.request.meta: response.request.meta['depth'] = 0 - stats.inc_value('request_depth_count/0', spider=spider) + self.stats.inc_value('request_depth_count/0', spider=spider) return (r for r in result or () if _filter(r)) diff --git a/scrapy/contrib/spidermiddleware/urllength.py b/scrapy/contrib/spidermiddleware/urllength.py index aa342a28d..dcc0d90be 100644 --- a/scrapy/contrib/spidermiddleware/urllength.py +++ b/scrapy/contrib/spidermiddleware/urllength.py @@ -7,13 +7,18 @@ See documentation in docs/topics/spider-middleware.rst from scrapy import log from scrapy.http import Request from scrapy.exceptions import NotConfigured -from scrapy.conf import settings class UrlLengthMiddleware(object): - def __init__(self): - self.maxlength = settings.getint('URLLENGTH_LIMIT') - if not self.maxlength: + + def __init__(self, maxlength): + self.maxlength = maxlength + + @classmethod + def from_settings(cls, settings): + maxlength = settings.getint('URLLENGTH_LIMIT') + if not maxlength: raise NotConfigured + return cls(maxlength) def process_spider_output(self, response, result, spider): def _filter(request): diff --git a/scrapy/tests/test_pipeline_images.py b/scrapy/tests/test_pipeline_images.py index 5d58abd2c..46f64d3a3 100644 --- a/scrapy/tests/test_pipeline_images.py +++ b/scrapy/tests/test_pipeline_images.py @@ -1,32 +1,29 @@ import os -from twisted.trial import unittest -from scrapy.crawler import Crawler -from scrapy.conf import settings from tempfile import mkdtemp from shutil import rmtree +from twisted.trial import unittest + +from scrapy.crawler import Crawler +from scrapy.conf import settings + class ImagesPipelineTestCase(unittest.TestCase): def setUp(self): - self.crawler = Crawler(settings) - self.crawler.install() try: import Image except ImportError, e: raise unittest.SkipTest(e) - from scrapy.contrib.pipeline.images import ImagesPipeline + + self.crawler = Crawler(settings) + self.crawler.install() self.tempdir = mkdtemp() - self.settings_disabled_before = settings.disabled - settings.disabled = False - settings.overrides['IMAGES_STORE'] = self.tempdir - self.pipeline = ImagesPipeline() + self.pipeline = ImagesPipeline(self.tempdir) def tearDown(self): - del self.pipeline - rmtree(self.tempdir) - settings.disabled = self.settings_disabled_before self.crawler.uninstall() + rmtree(self.tempdir) def test_image_path(self): image_path = self.pipeline.image_key diff --git a/scrapy/tests/test_spidermiddleware_depth.py b/scrapy/tests/test_spidermiddleware_depth.py index e52dffb94..3cb62ab25 100644 --- a/scrapy/tests/test_spidermiddleware_depth.py +++ b/scrapy/tests/test_spidermiddleware_depth.py @@ -1,25 +1,21 @@ from unittest import TestCase from scrapy.contrib.spidermiddleware.depth import DepthMiddleware -from scrapy.conf import settings from scrapy.http import Response, Request from scrapy.spider import BaseSpider -from scrapy.stats import stats +from scrapy.statscol import StatsCollector class TestDepthMiddleware(TestCase): def setUp(self): - settings.disabled = False - settings.overrides['DEPTH_LIMIT'] = 1 - settings.overrides['DEPTH_STATS'] = True - self.spider = BaseSpider('scrapytest.org') - stats.open_spider(self.spider) + self.stats = StatsCollector() + self.stats.open_spider(self.spider) - self.mw = DepthMiddleware() - self.assertEquals(stats.get_value('envinfo/request_depth_limit'), 1) + self.mw = DepthMiddleware(1, self.stats) + self.assertEquals(self.stats.get_value('envinfo/request_depth_limit'), 1) def test_process_spider_output(self): req = Request('http://scrapytest.org') @@ -30,7 +26,7 @@ class TestDepthMiddleware(TestCase): out = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEquals(out, result) - rdc = stats.get_value('request_depth_count/1', spider=self.spider) + rdc = self.stats.get_value('request_depth_count/1', spider=self.spider) self.assertEquals(rdc, 1) req.meta['depth'] = 1 @@ -38,13 +34,9 @@ class TestDepthMiddleware(TestCase): out2 = list(self.mw.process_spider_output(resp, result, self.spider)) self.assertEquals(out2, []) - rdm = stats.get_value('request_depth_max', spider=self.spider) + rdm = self.stats.get_value('request_depth_max', spider=self.spider) self.assertEquals(rdm, 1) def tearDown(self): - del settings.overrides['DEPTH_LIMIT'] - del settings.overrides['DEPTH_STATS'] - settings.disabled = True - - stats.close_spider(self.spider, '') + self.stats.close_spider(self.spider, '') diff --git a/scrapy/tests/test_spidermiddleware_urllength.py b/scrapy/tests/test_spidermiddleware_urllength.py index cbd80d488..e3d8c5474 100644 --- a/scrapy/tests/test_spidermiddleware_urllength.py +++ b/scrapy/tests/test_spidermiddleware_urllength.py @@ -1,6 +1,5 @@ from unittest import TestCase -from scrapy.conf import settings from scrapy.contrib.spidermiddleware.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spider import BaseSpider @@ -8,13 +7,6 @@ from scrapy.spider import BaseSpider class TestUrlLengthMiddleware(TestCase): - def setUp(self): - settings.disabled = False - settings.overrides['URLLENGTH_LIMIT'] = 25 - - self.spider = BaseSpider('foo') - self.mw = UrlLengthMiddleware() - def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -22,10 +14,8 @@ class TestUrlLengthMiddleware(TestCase): long_url_req = Request('http://scrapytest.org/this_is_a_long_url') reqs = [short_url_req, long_url_req] - out = list(self.mw.process_spider_output(res, reqs, self.spider)) + mw = UrlLengthMiddleware(maxlength=25) + spider = BaseSpider('foo') + out = list(mw.process_spider_output(res, reqs, spider)) self.assertEquals(out, [short_url_req]) - def tearDown(self): - del settings.overrides['URLLENGTH_LIMIT'] - settings.disabled = True -