diff --git a/docs/contributing.rst b/docs/contributing.rst index 3420eecac..9cced6efd 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -65,7 +65,7 @@ Well-written patches should: * include one (or more) test cases that check the bug fixed or the new functionality added. See `Writing tests`_ below. -* if you're adding or changing a public (documented) API, please include +* if you're adding or changing a public (documented) API, please include the documentation changes in the same patch. See `Documentation policies`_ below. @@ -73,7 +73,10 @@ Submitting patches ================== The best way to submit a patch is to issue a `pull request`_ on Github, -optionally creating a new issue first. +optionally creating a new issue first. And try to keep your commits to just +the new functionality; meaning, if you want to clean up the code, to conform +to :pep:`8` for example, make an additional commit. This keeps the commit +history cleaner. Alternatively, we also accept the patches in the traditional way of sending them to the `scrapy-developers`_ list. diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 51ba14dd0..3bc56b5c3 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -1,10 +1,11 @@ import unittest + from scrapy.spider import BaseSpider -from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url, url_has_any_extension -from scrapy.linkextractor import IGNORED_EXTENSIONS +from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url __doctests__ = ['scrapy.utils.url'] + class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self): @@ -48,10 +49,16 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) + spider = BaseSpider(name='example.com', allowed_domains=set(('example.com', 'example.net'))) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + + spider = BaseSpider(name='example.com', allowed_domains=('example.com', 'example.net')) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(BaseSpider): name = 'example.com' - allowed_domains = ['example.org', 'example.net'] + allowed_domains = ('example.org', 'example.net') self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider)) @@ -161,4 +168,3 @@ class UrlUtilsTest(unittest.TestCase): if __name__ == "__main__": unittest.main() - diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index bc8236e1a..dd7ca85ab 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,7 +5,6 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ - import urlparse import urllib import cgi @@ -13,6 +12,7 @@ import cgi from w3lib.url import * from scrapy.utils.python import unicode_to_str + def url_is_from_any_domain(url, domains): """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc @@ -22,15 +22,18 @@ def url_is_from_any_domain(url, domains): else: return False + def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, [spider.name] + \ - getattr(spider, 'allowed_domains', [])) + return url_is_from_any_domain(url, + [spider.name] + list(getattr(spider, 'allowed_domains', []))) + def url_has_any_extension(url, extensions): return posixpath.splitext(parse_url(url).path)[1].lower() in extensions -def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ + +def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, encoding=None): """Canonicalize the given url by applying the following procedures: @@ -70,6 +73,7 @@ def parse_url(url, encoding=None): return url if isinstance(url, urlparse.ParseResult) else \ urlparse.urlparse(unicode_to_str(url, encoding)) + def escape_ajax(url): """ Return the crawleable url according to: