From 454d5e57333e9f33c8d684e4e21f8f7e9493f310 Mon Sep 17 00:00:00 2001 From: Jana Cavojska Date: Sun, 26 Nov 2017 20:07:04 +0100 Subject: [PATCH] checking for subclass of URLWarning instead of checking error message text when URL in allowed_domains --- scrapy/spidermiddlewares/offsite.py | 7 ++++++- tests/test_spidermiddleware_offsite.py | 3 ++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 647792e5d..f595eef42 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -56,10 +56,15 @@ class OffsiteMiddleware(object): for domain in allowed_domains: url_pattern = re.compile("^https?://.*$") if url_pattern.match(domain): - warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, Warning) + warnings.warn("allowed_domains accepts only domains, not URLs. Ignoring URL entry %s in allowed_domains." % domain, URLWarning) + regex = r'^(.*\.)?(%s)$' % '|'.join(re.escape(d) for d in allowed_domains if d is not None) return re.compile(regex) def spider_opened(self, spider): self.host_regex = self.get_host_regex(spider) self.domains_seen = set() + + +class URLWarning(Warning): + pass \ No newline at end of file diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b532cc2ec..7e4af0d4c 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -5,6 +5,7 @@ from six.moves.urllib.parse import urlparse from scrapy.http import Response, Request from scrapy.spiders import Spider from scrapy.spidermiddlewares.offsite import OffsiteMiddleware +from scrapy.spidermiddlewares.offsite import URLWarning from scrapy.utils.test import get_crawler import warnings @@ -78,4 +79,4 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4): with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") self.mw.get_host_regex(self.spider) - assert "allowed_domains accepts only domains, not URLs." in str(w[-1].message) + assert issubclass(w[-1].category, URLWarning)