From 8ec3b476b03d6b8424f6dfc556758392e7a5a61f Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sun, 26 Nov 2017 16:36:15 +0100 Subject: [PATCH] triggering a warning when user puts URL in allowed_domains now covered by test --- scrapy/spidermiddlewares/offsite.py | 3 ++- tests/test_spidermiddleware_offsite.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 8ff35e29f..647792e5d 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -6,6 +6,7 @@ See documentation in docs/topics/spider-middleware.rst import re import logging +import warnings from scrapy import signals from scrapy.http import Request @@ -55,7 +56,7 @@ class OffsiteMiddleware(object): for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): - logger.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain) + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning) regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index 9ad86313c..b532cc2ec 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -6,6 +6,7 @@ from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware from scrapy.utils.test import get_crawler +import warnings class TestOffsiteMiddleware(TestCase): @@ -68,3 +69,13 @@ class TestOffsiteMiddleware4(TestOffsiteMiddleware3): reqs = [Request('http://scrapytest.org/1')] out = list(self.mw.process_spider_output(res, reqs, self.spider)) self.assertEqual(out, reqs) + + +class TestOffsiteMiddleware5(TestOffsiteMiddleware4): + + def test_get_host_regex(self): + self.spider.allowed_domains = ['http://scrapytest.org', 'scrapy.org', 'scrapy.test.org'] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + self.mw.get_host_regex(self.spider) + assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message)