mirror of https://github.com/scrapy/scrapy.git
Use from_settings for backward compatibility
This commit is contained in:
parent
6e5ea7924c
commit
d4b2b61255
|
|
@ -14,17 +14,15 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
class UrlLengthMiddleware:
|
||||
|
||||
def __init__(self, maxlength, stats):
|
||||
def __init__(self, maxlength):
|
||||
self.maxlength = maxlength
|
||||
self.stats = stats
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler):
|
||||
settings = crawler.settings
|
||||
def from_settings(cls, settings):
|
||||
maxlength = settings.getint('URLLENGTH_LIMIT')
|
||||
if not maxlength:
|
||||
raise NotConfigured
|
||||
return cls(maxlength, crawler.stats)
|
||||
return cls(maxlength)
|
||||
|
||||
def process_spider_output(self, response, result, spider):
|
||||
def _filter(request):
|
||||
|
|
@ -34,7 +32,7 @@ class UrlLengthMiddleware:
|
|||
{'maxlength': self.maxlength, 'url': request.url},
|
||||
extra={'spider': spider}
|
||||
)
|
||||
self.stats.inc_value('urllength/request_ignored_count', spider=spider)
|
||||
spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ from testfixtures import LogCapture
|
|||
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
|
||||
from scrapy.http import Response, Request
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.statscollectors import StatsCollector
|
||||
from scrapy.utils.test import get_crawler
|
||||
|
||||
|
||||
|
|
@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase):
|
|||
def setUp(self):
|
||||
crawler = get_crawler(Spider)
|
||||
self.spider = crawler._create_spider('foo')
|
||||
|
||||
self.stats = StatsCollector(crawler)
|
||||
self.stats.open_spider(self.spider)
|
||||
self.stats = self.spider.crawler.stats
|
||||
|
||||
self.maxlength = 25
|
||||
self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
|
||||
self.mw = UrlLengthMiddleware(maxlength=self.maxlength)
|
||||
|
||||
self.response = Response('http://scrapytest.org')
|
||||
self.short_url_req = Request('http://scrapytest.org/')
|
||||
self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
|
||||
self.reqs = [self.short_url_req, self.long_url_req]
|
||||
|
||||
def tearDown(self):
|
||||
self.stats.close_spider(self.spider, '')
|
||||
|
||||
def process_spider_output(self):
|
||||
return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue