RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Removed settings.disabled hack used in some tests. Closes #143

This commit is contained in:
Pablo Hoffman 2010-09-06 11:04:27 -03:00
parent 5f58af2005
commit 3a72e5c051
6 changed files with 68 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scrapy/contrib/pipeline/images.py
View File

@ -24,7 +24,6 @@ from scrapy.http import Request
from scrapy import signals
from scrapy.exceptions import DropItem, NotConfigured, IgnoreRequest
from scrapy.contrib.pipeline.media import MediaPipeline
from scrapy.conf import settings
class NoimagesDrop(DropItem):
@ -78,8 +77,8 @@ class FSImagesStore(object):
class S3ImagesStore(object):
AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
def __init__(self, uri):
assert uri.startswith('s3://')
@ -139,22 +138,33 @@ class ImagesPipeline(MediaPipeline):
"""
MEDIA_NAME = 'image'
MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0)
MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0)
EXPIRES = settings.getint('IMAGES_EXPIRES', 90)
THUMBS = settings.get('IMAGES_THUMBS', {})
MIN_WIDTH = 0
MIN_HEIGHT = 0
EXPIRES = 90
THUMBS = {}
STORE_SCHEMES = {
'': FSImagesStore,
'file': FSImagesStore,
's3': S3ImagesStore,
}
def __init__(self):
def __init__(self, store_uri):
self.store = self._get_store(store_uri)
super(ImagesPipeline, self).__init__()
@classmethod
def from_settings(cls, settings):
cls.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0)
cls.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0)
cls.EXPIRES = settings.getint('IMAGES_EXPIRES', 90)
cls.THUMBS = settings.get('IMAGES_THUMBS', {})
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
store_uri = settings['IMAGES_STORE']
if not store_uri:
raise NotConfigured
self.store = self._get_store(store_uri)
super(ImagesPipeline, self).__init__()
return cls(store_uri)
def _get_store(self, uri):
if os.path.isabs(uri): # to support win32 paths like: C:\\some\dir

28
scrapy/contrib/spidermiddleware/depth.py
View File

@ -6,16 +6,24 @@ See documentation in docs/topics/spider-middleware.rst
from scrapy import log
from scrapy.http import Request
from scrapy.stats import stats
from scrapy.conf import settings
class DepthMiddleware(object):
def __init__(self):
self.maxdepth = settings.getint('DEPTH_LIMIT')
self.stats = settings.getbool('DEPTH_STATS')
def __init__(self, maxdepth, stats=None):
self.maxdepth = maxdepth
self.stats = stats
if self.stats and self.maxdepth:
stats.set_value('envinfo/request_depth_limit', self.maxdepth)
stats.set_value('envinfo/request_depth_limit', maxdepth)
@classmethod
def from_settings(cls, settings):
maxdepth = settings.getint('DEPTH_LIMIT')
usestats = settings.getbool('DEPTH_STATS')
if usestats:
from scrapy.stats import stats
else:
stats = None
return cls(maxdepth, stats)
def process_spider_output(self, response, result, spider):
def _filter(request):
@ -27,14 +35,14 @@ class DepthMiddleware(object):
level=log.DEBUG, spider=spider)
return False
elif self.stats:
stats.inc_value('request_depth_count/%s' % depth, spider=spider)
if depth > stats.get_value('request_depth_max', 0, spider=spider):
stats.set_value('request_depth_max', depth, spider=spider)
self.stats.inc_value('request_depth_count/%s' % depth, spider=spider)
if depth > self.stats.get_value('request_depth_max', 0, spider=spider):
self.stats.set_value('request_depth_max', depth, spider=spider)
return True
# base case (depth=0)
if self.stats and 'depth' not in response.request.meta:
response.request.meta['depth'] = 0
stats.inc_value('request_depth_count/0', spider=spider)
self.stats.inc_value('request_depth_count/0', spider=spider)
return (r for r in result or () if _filter(r))

13
scrapy/contrib/spidermiddleware/urllength.py
View File

@ -7,13 +7,18 @@ See documentation in docs/topics/spider-middleware.rst
from scrapy import log
from scrapy.http import Request
from scrapy.exceptions import NotConfigured
from scrapy.conf import settings
class UrlLengthMiddleware(object):
def __init__(self):
self.maxlength = settings.getint('URLLENGTH_LIMIT')
if not self.maxlength:
def __init__(self, maxlength):
self.maxlength = maxlength
@classmethod
def from_settings(cls, settings):
maxlength = settings.getint('URLLENGTH_LIMIT')
if not maxlength:
raise NotConfigured
return cls(maxlength)
def process_spider_output(self, response, result, spider):
def _filter(request):

23
scrapy/tests/test_pipeline_images.py
View File

@ -1,32 +1,29 @@
import os
from twisted.trial import unittest
from scrapy.crawler import Crawler
from scrapy.conf import settings
from tempfile import mkdtemp
from shutil import rmtree
from twisted.trial import unittest
from scrapy.crawler import Crawler
from scrapy.conf import settings
class ImagesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.crawler = Crawler(settings)
self.crawler.install()
try:
import Image
except ImportError, e:
raise unittest.SkipTest(e)
from scrapy.contrib.pipeline.images import ImagesPipeline
self.crawler = Crawler(settings)
self.crawler.install()
self.tempdir = mkdtemp()
self.settings_disabled_before = settings.disabled
settings.disabled = False
settings.overrides['IMAGES_STORE'] = self.tempdir
self.pipeline = ImagesPipeline()
self.pipeline = ImagesPipeline(self.tempdir)
def tearDown(self):
del self.pipeline
rmtree(self.tempdir)
settings.disabled = self.settings_disabled_before
self.crawler.uninstall()
rmtree(self.tempdir)
def test_image_path(self):
image_path = self.pipeline.image_key

24
scrapy/tests/test_spidermiddleware_depth.py
View File

@ -1,25 +1,21 @@
from unittest import TestCase
from scrapy.contrib.spidermiddleware.depth import DepthMiddleware
from scrapy.conf import settings
from scrapy.http import Response, Request
from scrapy.spider import BaseSpider
from scrapy.stats import stats
from scrapy.statscol import StatsCollector
class TestDepthMiddleware(TestCase):
def setUp(self):
settings.disabled = False
settings.overrides['DEPTH_LIMIT'] = 1
settings.overrides['DEPTH_STATS'] = True
self.spider = BaseSpider('scrapytest.org')
stats.open_spider(self.spider)
self.stats = StatsCollector()
self.stats.open_spider(self.spider)
self.mw = DepthMiddleware()
self.assertEquals(stats.get_value('envinfo/request_depth_limit'), 1)
self.mw = DepthMiddleware(1, self.stats)
self.assertEquals(self.stats.get_value('envinfo/request_depth_limit'), 1)
def test_process_spider_output(self):
req = Request('http://scrapytest.org')
@ -30,7 +26,7 @@ class TestDepthMiddleware(TestCase):
out = list(self.mw.process_spider_output(resp, result, self.spider))
self.assertEquals(out, result)
rdc = stats.get_value('request_depth_count/1', spider=self.spider)
rdc = self.stats.get_value('request_depth_count/1', spider=self.spider)
self.assertEquals(rdc, 1)
req.meta['depth'] = 1
@ -38,13 +34,9 @@ class TestDepthMiddleware(TestCase):
out2 = list(self.mw.process_spider_output(resp, result, self.spider))
self.assertEquals(out2, [])
rdm = stats.get_value('request_depth_max', spider=self.spider)
rdm = self.stats.get_value('request_depth_max', spider=self.spider)
self.assertEquals(rdm, 1)
def tearDown(self):
del settings.overrides['DEPTH_LIMIT']
del settings.overrides['DEPTH_STATS']
settings.disabled = True
stats.close_spider(self.spider, '')
self.stats.close_spider(self.spider, '')

16
scrapy/tests/test_spidermiddleware_urllength.py
View File

@ -1,6 +1,5 @@
from unittest import TestCase
from scrapy.conf import settings
from scrapy.contrib.spidermiddleware.urllength import UrlLengthMiddleware
from scrapy.http import Response, Request
from scrapy.spider import BaseSpider
@ -8,13 +7,6 @@ from scrapy.spider import BaseSpider
class TestUrlLengthMiddleware(TestCase):
def setUp(self):
settings.disabled = False
settings.overrides['URLLENGTH_LIMIT'] = 25
self.spider = BaseSpider('foo')
self.mw = UrlLengthMiddleware()
def test_process_spider_output(self):
res = Response('http://scrapytest.org')
@ -22,10 +14,8 @@ class TestUrlLengthMiddleware(TestCase):
long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
reqs = [short_url_req, long_url_req]
out = list(self.mw.process_spider_output(res, reqs, self.spider))
mw = UrlLengthMiddleware(maxlength=25)
spider = BaseSpider('foo')
out = list(mw.process_spider_output(res, reqs, spider))
self.assertEquals(out, [short_url_req])
def tearDown(self):
del settings.overrides['URLLENGTH_LIMIT']
settings.disabled = True