Log skipped urls by length to INFO, add skipped stats

scrapy/spidermiddlewares/urllength.py

@@ -14,22 +14,27 @@ logger = logging.getLogger(__name__)
 
 class UrlLengthMiddleware:
 
-    def __init__(self, maxlength):
+    def __init__(self, maxlength, stats):
         self.maxlength = maxlength
+        self.stats = stats
 
     @classmethod
-    def from_settings(cls, settings):
+    def from_crawler(cls, crawler):
+        settings = crawler.settings
         maxlength = settings.getint('URLLENGTH_LIMIT')
         if not maxlength:
             raise NotConfigured
-        return cls(maxlength)
+        return cls(maxlength, crawler.stats)
 
     def process_spider_output(self, response, result, spider):
         def _filter(request):
             if isinstance(request, Request) and len(request.url) > self.maxlength:
-                logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ",
-                             {'maxlength': self.maxlength, 'url': request.url},
-                             extra={'spider': spider})
+                logger.info(
+                    "Ignoring link (url length > %(maxlength)d): %(url)s ",
+                    {'maxlength': self.maxlength, 'url': request.url},
+                    extra={'spider': spider}
+                )
+                self.stats.inc_value('urllength/request_ignored_count', spider=spider)
                 return False
             else:
                 return True

tests/test_spidermiddleware_urllength.py

@@ -1,20 +1,45 @@
 from unittest import TestCase
 
+from testfixtures import LogCapture
+
 from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
+from scrapy.utils.test import get_crawler
 
 
 class TestUrlLengthMiddleware(TestCase):
 
-    def test_process_spider_output(self):
-        res = Response('http://scrapytest.org')
+    def setUp(self):
+        crawler = get_crawler(Spider)
+        self.spider = crawler._create_spider('foo')
 
-        short_url_req = Request('http://scrapytest.org/')
-        long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
-        reqs = [short_url_req, long_url_req]
+        self.stats = StatsCollector(crawler)
+        self.stats.open_spider(self.spider)
 
-        mw = UrlLengthMiddleware(maxlength=25)
-        spider = Spider('foo')
-        out = list(mw.process_spider_output(res, reqs, spider))
-        self.assertEqual(out, [short_url_req])
+        self.maxlength = 25
+        self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
+
+        self.response = Response('http://scrapytest.org')
+        self.short_url_req = Request('http://scrapytest.org/')
+        self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
+        self.reqs = [self.short_url_req, self.long_url_req]
+
+    def tearDown(self):
+        self.stats.close_spider(self.spider, '')
+
+    def process_spider_output(self):
+        return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
+
+    def test_middleware_works(self):
+        self.assertEqual(self.process_spider_output(), [self.short_url_req])
+
+    def test_logging(self):
+        with LogCapture() as log:
+            self.process_spider_output()
+
+        ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider)
+        self.assertEqual(ric, 1)
+
+        self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log))