Merge pull request #261 from stav/allowed_domains

allow spider allowed_domains to be set/tuple, #259

docs/contributing.rst

@@ -65,7 +65,7 @@ Well-written patches should:
 * include one (or more) test cases that check the bug fixed or the new
   functionality added. See `Writing tests`_ below.
 
-* if you're adding or changing a public (documented) API, please include 
+* if you're adding or changing a public (documented) API, please include
   the documentation changes in the same patch.  See `Documentation policies`_
   below.
 
@@ -73,7 +73,10 @@ Submitting patches
 ==================
 
 The best way to submit a patch is to issue a `pull request`_ on Github,
-optionally creating a new issue first.
+optionally creating a new issue first. And try to keep your commits to just
+the new functionality; meaning, if you want to clean up the code, to conform
+to :pep:`8` for example, make an additional commit. This keeps the commit
+history cleaner.
 
 Alternatively, we also accept the patches in the traditional way of sending
 them to the `scrapy-developers`_ list.

scrapy/tests/test_utils_url.py

@@ -1,10 +1,11 @@
 import unittest
+
 from scrapy.spider import BaseSpider
-from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url, url_has_any_extension
-from scrapy.linkextractor import IGNORED_EXTENSIONS
+from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url
 
 __doctests__ = ['scrapy.utils.url']
 
+
 class UrlUtilsTest(unittest.TestCase):
 
     def test_url_is_from_any_domain(self):
@@ -48,10 +49,16 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider))
 
+        spider = BaseSpider(name='example.com', allowed_domains=set(('example.com', 'example.net')))
+        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
+
+        spider = BaseSpider(name='example.com', allowed_domains=('example.com', 'example.net'))
+        self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider))
+
     def test_url_is_from_spider_with_allowed_domains_class_attributes(self):
         class MySpider(BaseSpider):
             name = 'example.com'
-            allowed_domains = ['example.org', 'example.net']
+            allowed_domains = ('example.org', 'example.net')
         self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider))
         self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider))
         self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider))
@@ -161,4 +168,3 @@ class UrlUtilsTest(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
-

scrapy/utils/url.py

@@ -5,7 +5,6 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
-
 import urlparse
 import urllib
 import cgi
@@ -13,6 +12,7 @@ import cgi
 from w3lib.url import *
 from scrapy.utils.python import unicode_to_str
 
+
 def url_is_from_any_domain(url, domains):
     """Return True if the url belongs to any of the given domains"""
     host = parse_url(url).netloc
@@ -22,15 +22,18 @@ def url_is_from_any_domain(url, domains):
     else:
         return False
 
+
 def url_is_from_spider(url, spider):
     """Return True if the url belongs to the given spider"""
-    return url_is_from_any_domain(url, [spider.name] + \
-        getattr(spider, 'allowed_domains', []))
+    return url_is_from_any_domain(url,
+        [spider.name] + list(getattr(spider, 'allowed_domains', [])))
+
 
 def url_has_any_extension(url, extensions):
     return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
 
-def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \
+
+def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,
         encoding=None):
     """Canonicalize the given url by applying the following procedures:
 
@@ -70,6 +73,7 @@ def parse_url(url, encoding=None):
     return url if isinstance(url, urlparse.ParseResult) else \
         urlparse.urlparse(unicode_to_str(url, encoding))
 
+
 def escape_ajax(url):
     """
     Return the crawleable url according to: