Merge pull request #4410 from Lukas0907/fix-offsite-middleware-tests

Fix handling of None in allowed_domains

scrapy/spidermiddlewares/offsite.py

@@ -54,12 +54,16 @@ class OffsiteMiddleware(object):
         if not allowed_domains:
             return re.compile('')  # allow all by default
         url_pattern = re.compile("^https?://.*$")
+        domains = []
         for domain in allowed_domains:
-            if url_pattern.match(domain):
+            if domain is None:
+                continue
+            elif url_pattern.match(domain):
                 message = ("allowed_domains accepts only domains, not URLs. "
                            "Ignoring URL entry %s in allowed_domains." % domain)
                 warnings.warn(message, URLWarning)
-        domains = [re.escape(d) for d in allowed_domains if d is not None]
+            else:
+                domains.append(re.escape(domain))
         regex = r'^(.*\.)?(%s)$' % '|'.join(domains)
         return re.compile(regex)
 

tests/test_spidermiddleware_offsite.py

@@ -55,21 +55,21 @@ class TestOffsiteMiddleware2(TestOffsiteMiddleware):
 
 class TestOffsiteMiddleware3(TestOffsiteMiddleware2):
 
-    def _get_spider(self):
-        return Spider('foo')
+    def _get_spiderargs(self):
+        return dict(name='foo')
 
 
 class TestOffsiteMiddleware4(TestOffsiteMiddleware3):
 
-    def _get_spider(self):
-      bad_hostname = urlparse('http:////scrapytest.org').hostname
-      return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
+    def _get_spiderargs(self):
+        bad_hostname = urlparse('http:////scrapytest.org').hostname
+        return dict(name='foo', allowed_domains=['scrapytest.org', None, bad_hostname])
 
     def test_process_spider_output(self):
-      res = Response('http://scrapytest.org')
-      reqs = [Request('http://scrapytest.org/1')]
-      out = list(self.mw.process_spider_output(res, reqs, self.spider))
-      self.assertEqual(out, reqs)
+        res = Response('http://scrapytest.org')
+        reqs = [Request('http://scrapytest.org/1')]
+        out = list(self.mw.process_spider_output(res, reqs, self.spider))
+        self.assertEqual(out, reqs)
 
 
 class TestOffsiteMiddleware5(TestOffsiteMiddleware4):