diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 646a15968..49adac8bd 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -78,6 +78,6 @@ class RegexLinkExtractor(LinkExtractor): if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): return False - allowed = [regex.search(url) for regex in self.allow_res] - denied = [regex.search(url) for regex in self.deny_res] - return any(allowed) and not any(denied) + allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] + denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] + return True if any(allowed) and not any(denied) else False diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index 8c6b603d5..4fe992753 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -2,6 +2,7 @@ import unittest from scrapy.http import Response from scrapy.link import LinkExtractor, Link +from scrapy.link.extractors import RegexLinkExtractor class LinkExtractorTestCase(unittest.TestCase): def test_basic(self): @@ -31,5 +32,36 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_urls(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + def test_matches(self): + url1 = 'http://lotsofstuff.com/stuff1/index' + url2 = 'http://evenmorestuff.com/uglystuff/index' + + lx = LinkExtractor() + self.assertEqual(lx.matches(url1), True) + self.assertEqual(lx.matches(url2), True) + + lx = RegexLinkExtractor(allow=(r'stuff1', )) + self.assertEqual(lx.matches(url1), True) + self.assertEqual(lx.matches(url2), False) + + lx = RegexLinkExtractor(deny=(r'uglystuff', )) + self.assertEqual(lx.matches(url1), True) + self.assertEqual(lx.matches(url2), False) + + lx = RegexLinkExtractor(allow_domains=('evenmorestuff.com', )) + self.assertEqual(lx.matches(url1), False) + self.assertEqual(lx.matches(url2), True) + + lx = RegexLinkExtractor(deny_domains=('lotsofstuff.com', )) + self.assertEqual(lx.matches(url1), False) + self.assertEqual(lx.matches(url2), True) + + lx = RegexLinkExtractor(allow=('blah1', ), deny=('blah2', ), + allow_domains=('blah1.com', ), deny_domains=('blah2.com', )) + self.assertEqual(lx.matches('http://blah1.com/blah1'), True) + self.assertEqual(lx.matches('http://blah1.com/blah2'), False) + self.assertEqual(lx.matches('http://blah2.com/blah1'), False) + self.assertEqual(lx.matches('http://blah2.com/blah2'), False) + if __name__ == "__main__": unittest.main()