From 6e5ea7924c7e3f5dd958d13db73e04c6348c99ba Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 11:08:41 +0600 Subject: [PATCH 1/4] Log skipped urls by length to INFO, add skipped stats --- scrapy/spidermiddlewares/urllength.py | 17 ++++++---- tests/test_spidermiddleware_urllength.py | 43 +++++++++++++++++++----- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index 5be1f80cb..ee3cb9fd6 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,22 +14,27 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength): + def __init__(self, maxlength, stats): self.maxlength = maxlength + self.stats = stats @classmethod - def from_settings(cls, settings): + def from_crawler(cls, crawler): + settings = crawler.settings maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength) + return cls(maxlength, crawler.stats) def process_spider_output(self, response, result, spider): def _filter(request): if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ", - {'maxlength': self.maxlength, 'url': request.url}, - extra={'spider': spider}) + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {'maxlength': self.maxlength, 'url': request.url}, + extra={'spider': spider} + ) + self.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 5ef2b23fd..33c524627 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -1,20 +1,45 @@ from unittest import TestCase +from testfixtures import LogCapture + from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider +from scrapy.statscollectors import StatsCollector +from scrapy.utils.test import get_crawler class TestUrlLengthMiddleware(TestCase): - def test_process_spider_output(self): - res = Response('http://scrapytest.org') + def setUp(self): + crawler = get_crawler(Spider) + self.spider = crawler._create_spider('foo') - short_url_req = Request('http://scrapytest.org/') - long_url_req = Request('http://scrapytest.org/this_is_a_long_url') - reqs = [short_url_req, long_url_req] + self.stats = StatsCollector(crawler) + self.stats.open_spider(self.spider) - mw = UrlLengthMiddleware(maxlength=25) - spider = Spider('foo') - out = list(mw.process_spider_output(res, reqs, spider)) - self.assertEqual(out, [short_url_req]) + self.maxlength = 25 + self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + + self.response = Response('http://scrapytest.org') + self.short_url_req = Request('http://scrapytest.org/') + self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') + self.reqs = [self.short_url_req, self.long_url_req] + + def tearDown(self): + self.stats.close_spider(self.spider, '') + + def process_spider_output(self): + return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) + + def test_middleware_works(self): + self.assertEqual(self.process_spider_output(), [self.short_url_req]) + + def test_logging(self): + with LogCapture() as log: + self.process_spider_output() + + ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider) + self.assertEqual(ric, 1) + + self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log)) From d4b2b612551918647148013893da4cfa83fa2e7a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 16:59:37 +0600 Subject: [PATCH 2/4] Use from_settings for backward compatibility --- scrapy/spidermiddlewares/urllength.py | 10 ++++------ tests/test_spidermiddleware_urllength.py | 10 ++-------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index ee3cb9fd6..450d4ff40 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -14,17 +14,15 @@ logger = logging.getLogger(__name__) class UrlLengthMiddleware: - def __init__(self, maxlength, stats): + def __init__(self, maxlength): self.maxlength = maxlength - self.stats = stats @classmethod - def from_crawler(cls, crawler): - settings = crawler.settings + def from_settings(cls, settings): maxlength = settings.getint('URLLENGTH_LIMIT') if not maxlength: raise NotConfigured - return cls(maxlength, crawler.stats) + return cls(maxlength) def process_spider_output(self, response, result, spider): def _filter(request): @@ -34,7 +32,7 @@ class UrlLengthMiddleware: {'maxlength': self.maxlength, 'url': request.url}, extra={'spider': spider} ) - self.stats.inc_value('urllength/request_ignored_count', spider=spider) + spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider) return False else: return True diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 33c524627..6a72d2a8d 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -5,7 +5,6 @@ from testfixtures import LogCapture from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler @@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase): def setUp(self): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - - self.stats = StatsCollector(crawler) - self.stats.open_spider(self.spider) + self.stats = self.spider.crawler.stats self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats) + self.mw = UrlLengthMiddleware(maxlength=self.maxlength) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url') self.reqs = [self.short_url_req, self.long_url_req] - def tearDown(self): - self.stats.close_spider(self.spider, '') - def process_spider_output(self): return list(self.mw.process_spider_output(self.response, self.reqs, self.spider)) From 0f254a6afbc3ad4a42048ea67acafdd035ba690a Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Fri, 12 Mar 2021 17:11:50 +0600 Subject: [PATCH 3/4] Test from_settings --- tests/test_spidermiddleware_urllength.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index 6a72d2a8d..ee79c109f 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -6,17 +6,19 @@ from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.utils.test import get_crawler +from scrapy.settings import Settings class TestUrlLengthMiddleware(TestCase): def setUp(self): + self.maxlength = 25 + settings = Settings({'URLLENGTH_LIMIT': self.maxlength}) + crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') self.stats = self.spider.crawler.stats - - self.maxlength = 25 - self.mw = UrlLengthMiddleware(maxlength=self.maxlength) + self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org') self.short_url_req = Request('http://scrapytest.org/') From 9cc4513bd60dcebcbfc53035482eff82d2b7acc0 Mon Sep 17 00:00:00 2001 From: Dmitriy Pomazunovskiy Date: Mon, 15 Mar 2021 21:38:03 +0600 Subject: [PATCH 4/4] simpler stats access --- tests/test_spidermiddleware_urllength.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ee79c109f..171f4ddfd 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -17,7 +17,7 @@ class TestUrlLengthMiddleware(TestCase): crawler = get_crawler(Spider) self.spider = crawler._create_spider('foo') - self.stats = self.spider.crawler.stats + self.stats = crawler.stats self.mw = UrlLengthMiddleware.from_settings(settings) self.response = Response('http://scrapytest.org')