diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index 18a0f4487..7f2f4e4bc 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -1,12 +1,9 @@ """ SGMLParser-based Link extractors """ - import re from urlparse import urlparse, urljoin - from w3lib.url import safe_url_string - from scrapy.selector import HtmlXPathSelector from scrapy.link import Link from scrapy.linkextractor import IGNORED_EXTENSIONS @@ -15,6 +12,7 @@ from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_u from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension from scrapy.utils.response import get_base_url + class BaseSgmlLinkExtractor(FixedSGMLParser): def __init__(self, tag="a", attr="href", unique=False, process_value=None): @@ -92,9 +90,10 @@ _re_type = type(re.compile("", 0)) _matches = lambda url, regexs: any((r.search(url) for r in regexs)) _is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file']) + class SgmlLinkExtractor(BaseSgmlLinkExtractor): - def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), + def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None, deny_extensions=None): self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] @@ -108,8 +107,11 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): self.deny_extensions = set(['.' + e for e in deny_extensions]) tag_func = lambda x: x in tags attr_func = lambda x: x in attrs - BaseSgmlLinkExtractor.__init__(self, tag=tag_func, attr=attr_func, - unique=unique, process_value=process_value) + BaseSgmlLinkExtractor.__init__(self, + tag=tag_func, + attr=attr_func, + unique=unique, + process_value=process_value) def extract_links(self, response): base_url = None diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index d1992b170..59147035a 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -1,11 +1,11 @@ import re import unittest - from scrapy.http import HtmlResponse from scrapy.link import Link from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor from scrapy.tests import get_testdata + class LinkExtractorTestCase(unittest.TestCase): def test_basic(self): html = """Page title<title> @@ -20,9 +20,9 @@ class LinkExtractorTestCase(unittest.TestCase): lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), + [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), Link(url='http://example.org/about.html', text='About us'), - Link(url='http://example.org/othercat.html', text='Other category'), + Link(url='http://example.org/othercat.html', text='Other category'), Link(url='http://example.org/', text='>>'), Link(url='http://example.org/', text='')]) @@ -54,8 +54,9 @@ class LinkExtractorTestCase(unittest.TestCase): html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>""" response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8') lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response), - [Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd')]) + self.assertEqual(lx.extract_links(response), [ + Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'), + ]) def test_extraction_encoding(self): body = get_testdata('link_extractor', 'linkextractor_noenc.html') @@ -65,17 +66,20 @@ class LinkExtractorTestCase(unittest.TestCase): response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) lx = BaseSgmlLinkExtractor() - self.assertEqual(lx.extract_links(response_utf8), - [ Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ]) + self.assertEqual(lx.extract_links(response_utf8), [ + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), + ]) - self.assertEqual(lx.extract_links(response_noenc), - [ Link(url='http://example.com/sample_%C3%B1.html', text=''), - Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ]) + self.assertEqual(lx.extract_links(response_noenc), [ + Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')), + ]) - self.assertEqual(lx.extract_links(response_latin1), - [ Link(url='http://example.com/sample_%F1.html', text=''), - Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')) ]) + self.assertEqual(lx.extract_links(response_latin1), [ + Link(url='http://example.com/sample_%F1.html', text=''), + Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')), + ]) def test_matches(self): url1 = 'http://lotsofstuff.com/stuff1/index' @@ -92,9 +96,11 @@ class LinkExtractorTestCase(unittest.TestCase): """ response = HtmlResponse("http://example.org/page.html", body=html) lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(response)], - [ Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), - Link(url='http://example.org/about.html', text=u'About us', nofollow=False) ]) + self.assertEqual([link for link in lx.extract_links(response)], [ + Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True), + Link(url='http://example.org/about.html', text=u'About us', nofollow=False), + ]) + class SgmlLinkExtractorTestCase(unittest.TestCase): def setUp(self): @@ -110,62 +116,71 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): '''Test the extractor's behaviour among different situations''' lx = SgmlLinkExtractor() - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://www.google.com/something', text=u'') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://www.google.com/something', text=u''), + ]) lx = SgmlLinkExtractor(allow=('sample', )) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) lx = SgmlLinkExtractor(allow=('sample', ), unique=False) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://example.com/sample3.html', text=u'sample 3 repetition') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), + ]) lx = SgmlLinkExtractor(allow=('sample', )) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', )) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) lx = SgmlLinkExtractor(allow_domains=('google.com', )) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://www.google.com/something', text=u'') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://www.google.com/something', text=u''), + ]) def test_extraction_using_single_values(self): '''Test the extractor's behaviour among different situations''' lx = SgmlLinkExtractor(allow='sample') - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2'), - Link(url='http://example.com/sample3.html', text=u'sample 3 text') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + Link(url='http://example.com/sample3.html', text=u'sample 3 text'), + ]) lx = SgmlLinkExtractor(allow='sample', deny='3') - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) lx = SgmlLinkExtractor(allow_domains='google.com') - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://www.google.com/something', text=u'') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://www.google.com/something', text=u''), + ]) lx = SgmlLinkExtractor(deny_domains='example.com') - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://www.google.com/something', text=u'') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://www.google.com/something', text=u''), + ]) def test_matches(self): url1 = 'http://lotsofstuff.com/stuff1/index' @@ -187,8 +202,9 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = SgmlLinkExtractor(allow=('blah1', ), deny=('blah2', ), - allow_domains=('blah1.com', ), deny_domains=('blah2.com', )) + lx = SgmlLinkExtractor(allow=('blah1',), deny=('blah2',), + allow_domains=('blah1.com',), + deny_domains=('blah2.com',)) self.assertEqual(lx.matches('http://blah1.com/blah1'), True) self.assertEqual(lx.matches('http://blah1.com/blah2'), False) self.assertEqual(lx.matches('http://blah2.com/blah1'), False) @@ -196,9 +212,10 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): def test_restrict_xpaths(self): lx = SgmlLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', )) - self.assertEqual([link for link in lx.extract_links(self.response)], - [ Link(url='http://example.com/sample1.html', text=u''), - Link(url='http://example.com/sample2.html', text=u'sample 2') ]) + self.assertEqual([link for link in lx.extract_links(self.response)], [ + Link(url='http://example.com/sample1.html', text=u''), + Link(url='http://example.com/sample2.html', text=u'sample 2'), + ]) def test_restrict_xpaths_encoding(self): """Test restrict_xpaths with encodings""" @@ -213,7 +230,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') - lx = SgmlLinkExtractor(restrict_xpaths="//div[@class='links']") + lx = SgmlLinkExtractor(restrict_xpaths="//div[@class='links']") self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/about.html', text=u'About us\xa3')]) @@ -230,8 +247,9 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" response = HtmlResponse("http://example.org/", body=html) lx = SgmlLinkExtractor() - self.assertEqual(lx.extract_links(response), - [Link(url='http://example.org/page.html', text=u'asd')]) + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/page.html', text=u'asd'), + ]) def test_process_value(self): """Test restrict_xpaths with encodings""" @@ -255,11 +273,10 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): <body><p><a href="item/12.html">Item 12</a></p> </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html) - lx = SgmlLinkExtractor(restrict_xpaths="//p") + lx = SgmlLinkExtractor(restrict_xpaths="//p") self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) - if __name__ == "__main__": unittest.main()