mirror of https://github.com/scrapy/scrapy.git
pep8ize sgml link extractors
This commit is contained in:
Daniel Graña
parent
cc69b3aa4c
commit
872a22df68
|
|
@ -1,12 +1,9 @@
|
|||
"""
|
||||
SGMLParser-based Link extractors
|
||||
"""
|
||||
|
||||
import re
|
||||
from urlparse import urlparse, urljoin
|
||||
|
||||
from w3lib.url import safe_url_string
|
||||
|
||||
from scrapy.selector import HtmlXPathSelector
|
||||
from scrapy.link import Link
|
||||
from scrapy.linkextractor import IGNORED_EXTENSIONS
|
||||
|
|
@ -15,6 +12,7 @@ from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_u
|
|||
from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension
|
||||
from scrapy.utils.response import get_base_url
|
||||
|
||||
|
||||
class BaseSgmlLinkExtractor(FixedSGMLParser):
|
||||
|
||||
def __init__(self, tag="a", attr="href", unique=False, process_value=None):
|
||||
|
|
@ -92,9 +90,10 @@ _re_type = type(re.compile("", 0))
|
|||
_matches = lambda url, regexs: any((r.search(url) for r in regexs))
|
||||
_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file'])
|
||||
|
||||
|
||||
class SgmlLinkExtractor(BaseSgmlLinkExtractor):
|
||||
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(),
|
||||
tags=('a', 'area'), attrs=('href'), canonicalize=True, unique=True, process_value=None,
|
||||
deny_extensions=None):
|
||||
self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)]
|
||||
|
|
@ -108,8 +107,11 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor):
|
|||
self.deny_extensions = set(['.' + e for e in deny_extensions])
|
||||
tag_func = lambda x: x in tags
|
||||
attr_func = lambda x: x in attrs
|
||||
BaseSgmlLinkExtractor.__init__(self, tag=tag_func, attr=attr_func,
|
||||
unique=unique, process_value=process_value)
|
||||
BaseSgmlLinkExtractor.__init__(self,
|
||||
tag=tag_func,
|
||||
attr=attr_func,
|
||||
unique=unique,
|
||||
process_value=process_value)
|
||||
|
||||
def extract_links(self, response):
|
||||
base_url = None
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
import re
|
||||
import unittest
|
||||
|
||||
from scrapy.http import HtmlResponse
|
||||
from scrapy.link import Link
|
||||
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor
|
||||
from scrapy.tests import get_testdata
|
||||
|
||||
|
||||
class LinkExtractorTestCase(unittest.TestCase):
|
||||
def test_basic(self):
|
||||
html = """<html><head><title>Page title<title>
|
||||
|
|
@ -20,9 +20,9 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
|
||||
lx = BaseSgmlLinkExtractor() # default: tag=a, attr=href
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
|
||||
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
|
||||
Link(url='http://example.org/about.html', text='About us'),
|
||||
Link(url='http://example.org/othercat.html', text='Other category'),
|
||||
Link(url='http://example.org/othercat.html', text='Other category'),
|
||||
Link(url='http://example.org/', text='>>'),
|
||||
Link(url='http://example.org/', text='')])
|
||||
|
||||
|
|
@ -54,8 +54,9 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
html = """<body><p><a href="item/12.html">Wrong: \xed</a></p></body></html>"""
|
||||
response = HtmlResponse("http://www.example.com", body=html, encoding='utf-8')
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd')])
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://www.example.com/item/12.html', text=u'Wrong: \ufffd'),
|
||||
])
|
||||
|
||||
def test_extraction_encoding(self):
|
||||
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
|
||||
|
|
@ -65,17 +66,20 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
|
||||
|
||||
lx = BaseSgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response_utf8),
|
||||
[ Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ])
|
||||
self.assertEqual(lx.extract_links(response_utf8), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_noenc),
|
||||
[ Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ])
|
||||
self.assertEqual(lx.extract_links(response_noenc), [
|
||||
Link(url='http://example.com/sample_%C3%B1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')),
|
||||
])
|
||||
|
||||
self.assertEqual(lx.extract_links(response_latin1),
|
||||
[ Link(url='http://example.com/sample_%F1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')) ])
|
||||
self.assertEqual(lx.extract_links(response_latin1), [
|
||||
Link(url='http://example.com/sample_%F1.html', text=''),
|
||||
Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')),
|
||||
])
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
|
|
@ -92,9 +96,11 @@ class LinkExtractorTestCase(unittest.TestCase):
|
|||
"""
|
||||
response = HtmlResponse("http://example.org/page.html", body=html)
|
||||
lx = SgmlLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(response)],
|
||||
[ Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
|
||||
Link(url='http://example.org/about.html', text=u'About us', nofollow=False) ])
|
||||
self.assertEqual([link for link in lx.extract_links(response)], [
|
||||
Link(url='http://example.org/page.html?action=print', text=u'Printer-friendly page', nofollow=True),
|
||||
Link(url='http://example.org/about.html', text=u'About us', nofollow=False),
|
||||
])
|
||||
|
||||
|
||||
class SgmlLinkExtractorTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
|
@ -110,62 +116,71 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
'''Test the extractor's behaviour among different situations'''
|
||||
|
||||
lx = SgmlLinkExtractor()
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u'') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow=('sample', ), unique=False)
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow=('sample', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow_domains=('google.com', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://www.google.com/something', text=u'') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
def test_extraction_using_single_values(self):
|
||||
'''Test the extractor's behaviour among different situations'''
|
||||
|
||||
lx = SgmlLinkExtractor(allow='sample')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow='sample', deny='3')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(allow_domains='google.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://www.google.com/something', text=u'') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
lx = SgmlLinkExtractor(deny_domains='example.com')
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://www.google.com/something', text=u'') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://www.google.com/something', text=u''),
|
||||
])
|
||||
|
||||
def test_matches(self):
|
||||
url1 = 'http://lotsofstuff.com/stuff1/index'
|
||||
|
|
@ -187,8 +202,9 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
self.assertEqual(lx.matches(url1), False)
|
||||
self.assertEqual(lx.matches(url2), True)
|
||||
|
||||
lx = SgmlLinkExtractor(allow=('blah1', ), deny=('blah2', ),
|
||||
allow_domains=('blah1.com', ), deny_domains=('blah2.com', ))
|
||||
lx = SgmlLinkExtractor(allow=('blah1',), deny=('blah2',),
|
||||
allow_domains=('blah1.com',),
|
||||
deny_domains=('blah2.com',))
|
||||
self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
|
||||
self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
|
||||
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
|
||||
|
|
@ -196,9 +212,10 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
|
||||
def test_restrict_xpaths(self):
|
||||
lx = SgmlLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', ))
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)],
|
||||
[ Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2') ])
|
||||
self.assertEqual([link for link in lx.extract_links(self.response)], [
|
||||
Link(url='http://example.com/sample1.html', text=u''),
|
||||
Link(url='http://example.com/sample2.html', text=u'sample 2'),
|
||||
])
|
||||
|
||||
def test_restrict_xpaths_encoding(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
|
|
@ -213,7 +230,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252')
|
||||
|
||||
lx = SgmlLinkExtractor(restrict_xpaths="//div[@class='links']")
|
||||
lx = SgmlLinkExtractor(restrict_xpaths="//div[@class='links']")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/about.html', text=u'About us\xa3')])
|
||||
|
||||
|
|
@ -230,8 +247,9 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
html = """<a href="page.html">asd</a> and <a href="photo.jpg">"""
|
||||
response = HtmlResponse("http://example.org/", body=html)
|
||||
lx = SgmlLinkExtractor()
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://example.org/page.html', text=u'asd')])
|
||||
self.assertEqual(lx.extract_links(response), [
|
||||
Link(url='http://example.org/page.html', text=u'asd'),
|
||||
])
|
||||
|
||||
def test_process_value(self):
|
||||
"""Test restrict_xpaths with encodings"""
|
||||
|
|
@ -255,11 +273,10 @@ class SgmlLinkExtractorTestCase(unittest.TestCase):
|
|||
<body><p><a href="item/12.html">Item 12</a></p>
|
||||
</body></html>"""
|
||||
response = HtmlResponse("http://example.org/somepage/index.html", body=html)
|
||||
lx = SgmlLinkExtractor(restrict_xpaths="//p")
|
||||
lx = SgmlLinkExtractor(restrict_xpaths="//p")
|
||||
self.assertEqual(lx.extract_links(response),
|
||||
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue