Ignore a domain in allowed_domains with port and issue a warning (#4413)

scrapy/spidermiddlewares/offsite.py

@@ -53,7 +53,8 @@ class OffsiteMiddleware(object):
         allowed_domains = getattr(spider, 'allowed_domains', None)
         if not allowed_domains:
             return re.compile('')  # allow all by default
-        url_pattern = re.compile("^https?://.*$")
+        url_pattern = re.compile(r"^https?://.*$")
+        port_pattern = re.compile(r":\d+$")
         domains = []
         for domain in allowed_domains:
             if domain is None:
@@ -62,6 +63,10 @@ class OffsiteMiddleware(object):
                 message = ("allowed_domains accepts only domains, not URLs. "
                            "Ignoring URL entry %s in allowed_domains." % domain)
                 warnings.warn(message, URLWarning)
+            elif port_pattern.search(domain):
+                message = ("allowed_domains accepts only domains without ports. "
+                           "Ignoring entry %s in allowed_domains." % domain)
+                warnings.warn(message, PortWarning)
             else:
                 domains.append(re.escape(domain))
         regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
@@ -74,3 +79,7 @@ class OffsiteMiddleware(object):
 
 class URLWarning(Warning):
     pass
+
+
+class PortWarning(Warning):
+    pass

tests/test_spidermiddleware_offsite.py

@@ -4,7 +4,7 @@ import warnings
 
 from scrapy.http import Response, Request
 from scrapy.spiders import Spider
-from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning
+from scrapy.spidermiddlewares.offsite import OffsiteMiddleware, URLWarning, PortWarning
 from scrapy.utils.test import get_crawler
 
 
@@ -26,7 +26,8 @@ class TestOffsiteMiddleware(TestCase):
                        Request('http://scrapy.org/1'),
                        Request('http://sub.scrapy.org/1'),
                        Request('http://offsite.tld/letmepass', dont_filter=True),
-                       Request('http://scrapy.test.org/')]
+                       Request('http://scrapy.test.org/'),
+                       Request('http://scrapy.test.org:8000/')]
         offsite_reqs = [Request('http://scrapy2.org'),
                        Request('http://offsite.tld/'),
                        Request('http://offsite.tld/scrapytest.org'),
@@ -80,3 +81,13 @@ class TestOffsiteMiddleware5(TestOffsiteMiddleware4):
             warnings.simplefilter("always")
             self.mw.get_host_regex(self.spider)
             assert issubclass(w[-1].category, URLWarning)
+
+
+class TestOffsiteMiddleware6(TestOffsiteMiddleware4):
+
+    def test_get_host_regex(self):
+        self.spider.allowed_domains = ['scrapytest.org:8000', 'scrapy.org', 'scrapy.test.org']
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter("always")
+            self.mw.get_host_regex(self.spider)
+            assert issubclass(w[-1].category, PortWarning)