From b48ec1dce4c02c36ad6037dc34355f0c7705c807 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Wed, 6 Mar 2013 00:19:47 -0600 Subject: [PATCH 1/5] allow spider allowed_domains to be set/tuple, #259 --- scrapy/utils/url.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index bc8236e1a..dd7ca85ab 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -5,7 +5,6 @@ library. Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ - import urlparse import urllib import cgi @@ -13,6 +12,7 @@ import cgi from w3lib.url import * from scrapy.utils.python import unicode_to_str + def url_is_from_any_domain(url, domains): """Return True if the url belongs to any of the given domains""" host = parse_url(url).netloc @@ -22,15 +22,18 @@ def url_is_from_any_domain(url, domains): else: return False + def url_is_from_spider(url, spider): """Return True if the url belongs to the given spider""" - return url_is_from_any_domain(url, [spider.name] + \ - getattr(spider, 'allowed_domains', [])) + return url_is_from_any_domain(url, + [spider.name] + list(getattr(spider, 'allowed_domains', []))) + def url_has_any_extension(url, extensions): return posixpath.splitext(parse_url(url).path)[1].lower() in extensions -def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ + +def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, encoding=None): """Canonicalize the given url by applying the following procedures: @@ -70,6 +73,7 @@ def parse_url(url, encoding=None): return url if isinstance(url, urlparse.ParseResult) else \ urlparse.urlparse(unicode_to_str(url, encoding)) + def escape_ajax(url): """ Return the crawleable url according to: From a613e15154970dde12455ede88bee19f3b437d88 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Sun, 10 Mar 2013 16:41:27 -0600 Subject: [PATCH 2/5] add tests for url_is_from_spider() with allowed_domains --- scrapy/tests/test_utils_url.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 51ba14dd0..594514dac 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -48,6 +48,16 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', spider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', spider)) + spider = BaseSpider(name='example.com', allowed_domains=set(('example.com', 'example.net'))) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + + spider = BaseSpider(name='example.com', allowed_domains=('example.com', 'example.net')) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) + self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider)) + + spider = BaseSpider(name='example.com', allowed_domains={'example.org': None}) + self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider)) + def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(BaseSpider): name = 'example.com' @@ -59,6 +69,16 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) + class MySetSpider(BaseSpider): + name = 'example.com' + allowed_domains = set(('example.org', 'example.net')) + self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySetSpider)) + self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySetSpider)) + self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySetSpider)) + self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySetSpider)) + self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySetSpider)) + self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySetSpider)) + def test_canonicalize_url(self): # simplest case self.assertEqual(canonicalize_url("http://www.example.com/"), From 1514b3b5db1183c97c2379cfc1b2a1710ba0f598 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Sun, 10 Mar 2013 16:47:24 -0600 Subject: [PATCH 3/5] pylint clean-ups for test_utils_url.py --- scrapy/tests/test_utils_url.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index 594514dac..deda64398 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -1,10 +1,11 @@ import unittest + from scrapy.spider import BaseSpider -from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url, url_has_any_extension -from scrapy.linkextractor import IGNORED_EXTENSIONS +from scrapy.utils.url import url_is_from_any_domain, url_is_from_spider, canonicalize_url __doctests__ = ['scrapy.utils.url'] + class UrlUtilsTest(unittest.TestCase): def test_url_is_from_any_domain(self): @@ -181,4 +182,3 @@ class UrlUtilsTest(unittest.TestCase): if __name__ == "__main__": unittest.main() - From 5828179c5f76db0b98a44db3702a3465122e1159 Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Sun, 10 Mar 2013 18:30:49 -0600 Subject: [PATCH 4/5] remove over-testing and dict testing for test_utils_url.py --- scrapy/tests/test_utils_url.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index deda64398..3bc56b5c3 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -54,15 +54,11 @@ class UrlUtilsTest(unittest.TestCase): spider = BaseSpider(name='example.com', allowed_domains=('example.com', 'example.net')) self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', spider)) - self.assertFalse(url_is_from_spider('http://www.example.org/some/page.html', spider)) - - spider = BaseSpider(name='example.com', allowed_domains={'example.org': None}) - self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', spider)) def test_url_is_from_spider_with_allowed_domains_class_attributes(self): class MySpider(BaseSpider): name = 'example.com' - allowed_domains = ['example.org', 'example.net'] + allowed_domains = ('example.org', 'example.net') self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySpider)) self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySpider)) @@ -70,16 +66,6 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) - class MySetSpider(BaseSpider): - name = 'example.com' - allowed_domains = set(('example.org', 'example.net')) - self.assertTrue(url_is_from_spider('http://www.example.com/some/page.html', MySetSpider)) - self.assertTrue(url_is_from_spider('http://sub.example.com/some/page.html', MySetSpider)) - self.assertTrue(url_is_from_spider('http://example.com/some/page.html', MySetSpider)) - self.assertTrue(url_is_from_spider('http://www.example.org/some/page.html', MySetSpider)) - self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySetSpider)) - self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySetSpider)) - def test_canonicalize_url(self): # simplest case self.assertEqual(canonicalize_url("http://www.example.com/"), From 650eda68da1e835b72bc78b2f1fa8cae7e4acdab Mon Sep 17 00:00:00 2001 From: Steven Almeroth Date: Sun, 10 Mar 2013 18:51:04 -0600 Subject: [PATCH 5/5] doc: add comment about commit history cleanliness --- docs/contributing.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 3420eecac..9cced6efd 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -65,7 +65,7 @@ Well-written patches should: * include one (or more) test cases that check the bug fixed or the new functionality added. See `Writing tests`_ below. -* if you're adding or changing a public (documented) API, please include +* if you're adding or changing a public (documented) API, please include the documentation changes in the same patch. See `Documentation policies`_ below. @@ -73,7 +73,10 @@ Submitting patches ================== The best way to submit a patch is to issue a `pull request`_ on Github, -optionally creating a new issue first. +optionally creating a new issue first. And try to keep your commits to just +the new functionality; meaning, if you want to clean up the code, to conform +to :pep:`8` for example, make an additional commit. This keeps the commit +history cleaner. Alternatively, we also accept the patches in the traditional way of sending them to the `scrapy-developers`_ list.