diff --git a/scrapy/contrib/linkextractors/lxmlhtml.py b/scrapy/contrib/linkextractors/lxmlhtml.py index 826c76227..fb5ac2db3 100644 --- a/scrapy/contrib/linkextractors/lxmlhtml.py +++ b/scrapy/contrib/linkextractors/lxmlhtml.py @@ -2,10 +2,30 @@ Link extractor based on lxml.html """ -import lxml.html +import re +from urlparse import urlparse, urljoin +import lxml.etree as etree + +from scrapy.selector import Selector from scrapy.link import Link -from scrapy.utils.python import unique as unique_list +from scrapy.utils.misc import arg_to_iter +from scrapy.utils.python import unique as unique_list, str_to_unicode +from scrapy.linkextractor import FilteringLinkExtractor +from scrapy.utils.response import get_base_url + + +# from lxml/src/lxml/html/__init__.py +XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml" + +_collect_string_content = etree.XPath("string()") + +def _nons(tag): + if isinstance(tag, basestring): + if tag[0] == '{' and tag[1:len(XHTML_NAMESPACE)+1] == XHTML_NAMESPACE: + return tag.split('}')[-1] + return tag + class LxmlParserLinkExtractor(object): def __init__(self, tag="a", attr="href", process=None, unique=False): @@ -16,14 +36,31 @@ class LxmlParserLinkExtractor(object): self.links = [] - def _extract_links(self, response_text, response_url): - html = lxml.html.fromstring(response_text) - html.make_links_absolute(response_url) - for e, a, l, p in html.iterlinks(): - if self.scan_tag(e.tag): - if self.scan_attr(a): - link = Link(self.process_attr(l), text=e.text) - self.links.append(link) + def _iter_links(self, document): + for el in document.iter(etree.Element): + tag = _nons(el.tag) + if not self.scan_tag(el.tag): + continue + attribs = el.attrib + for attrib in attribs: + yield (el, attrib, attribs[attrib]) + + def _extract_links(self, selector, response_url, response_encoding, base_url): + # hacky way to get the underlying lxml parsed document + for el, attr, attr_val in self._iter_links(selector._root): + if self.scan_tag(el.tag) and self.scan_attr(attr): + # pseudo _root.make_links_absolute(base_url) + attr_val = urljoin(base_url, attr_val) + url = self.process_attr(attr_val) + if url is None: + continue + if isinstance(url, unicode): + url = url.encode(response_encoding) + # to fix relative links after process_value + url = urljoin(response_url, url) + link = Link(url, _collect_string_content(el) or u'', + nofollow=True if el.get('rel') == 'nofollow' else False) + self.links.append(link) links = unique_list(self.links, key=lambda link: link.url) \ if self.unique else self.links @@ -31,6 +68,46 @@ class LxmlParserLinkExtractor(object): return links def extract_links(self, response): - return self._extract_links(response.body, response.url) + html = Selector(response) + base_url = get_base_url(response) + return self._extract_links(html, response.url, response.encoding, base_url) + + def _process_links(self, links): + """ Normalize and filter extracted links + + The subclass should override it if neccessary + """ + links = unique_list(links, key=lambda link: link.url) if self.unique else links + return links +class LxmlLinkExtractor(FilteringLinkExtractor): + + def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), + tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, + deny_extensions=None): + tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + lx = LxmlParserLinkExtractor(tag=tag_func, attr=attr_func, + unique=unique, process=process_value) + + super(LxmlLinkExtractor, self).__init__(lx, allow, deny, + allow_domains, deny_domains, restrict_xpaths, canonicalize, + deny_extensions) + + def extract_links(self, response): + html = Selector(response) + base_url = get_base_url(response) + if self.restrict_xpaths: + docs = [subdoc + for x in self.restrict_xpaths + for subdoc in html.xpath(x)] + else: + docs = [html] + all_links = [] + for doc in docs: + links = self._extract_links(doc, response.url, response.encoding, base_url) + all_links.extend(self._process_links(links)) + return unique_list(all_links) + diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index a0b51b218..9adaad9e5 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -6,10 +6,9 @@ from urlparse import urlparse, urljoin from w3lib.url import safe_url_string from scrapy.selector import Selector from scrapy.link import Link -from scrapy.linkextractor import IGNORED_EXTENSIONS +from scrapy.linkextractor import FilteringLinkExtractor from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import FixedSGMLParser, unique as unique_list, str_to_unicode -from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension from scrapy.utils.response import get_base_url @@ -86,33 +85,23 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): it doesn't contain any patterns""" return True -_re_type = type(re.compile("", 0)) -_matches = lambda url, regexs: any((r.search(url) for r in regexs)) -_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file']) - - -class SgmlLinkExtractor(BaseSgmlLinkExtractor): +class SgmlLinkExtractor(FilteringLinkExtractor): def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(), restrict_xpaths=(), tags=('a', 'area'), attrs=('href',), canonicalize=True, unique=True, process_value=None, deny_extensions=None): - self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] - self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] - self.allow_domains = set(arg_to_iter(allow_domains)) - self.deny_domains = set(arg_to_iter(deny_domains)) - self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) - self.canonicalize = canonicalize - if deny_extensions is None: - deny_extensions = IGNORED_EXTENSIONS - self.deny_extensions = {'.' + e for e in arg_to_iter(deny_extensions)} - tag_func = lambda x: x in arg_to_iter(tags) - attr_func = lambda x: x in arg_to_iter(attrs) - BaseSgmlLinkExtractor.__init__(self, - tag=tag_func, - attr=attr_func, - unique=unique, - process_value=process_value) + tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs)) + tag_func = lambda x: x in tags + attr_func = lambda x: x in attrs + lx = BaseSgmlLinkExtractor(tag=tag_func, attr=attr_func, + unique=unique, process_value=process_value) + super(SgmlLinkExtractor, self).__init__(lx, allow, deny, + allow_domains, deny_domains, restrict_xpaths, canonicalize, + deny_extensions) + + # FIXME: was added to fix a RegexLinkExtractor testcase + self.base_url = None def extract_links(self, response): base_url = None @@ -129,35 +118,3 @@ class SgmlLinkExtractor(BaseSgmlLinkExtractor): links = self._extract_links(body, response.url, response.encoding, base_url) links = self._process_links(links) return links - - def _process_links(self, links): - links = [x for x in links if self._link_allowed(x)] - links = BaseSgmlLinkExtractor._process_links(self, links) - return links - - def _link_allowed(self, link): - parsed_url = urlparse(link.url) - allowed = _is_valid_url(link.url) - if self.allow_res: - allowed &= _matches(link.url, self.allow_res) - if self.deny_res: - allowed &= not _matches(link.url, self.deny_res) - if self.allow_domains: - allowed &= url_is_from_any_domain(parsed_url, self.allow_domains) - if self.deny_domains: - allowed &= not url_is_from_any_domain(parsed_url, self.deny_domains) - if self.deny_extensions: - allowed &= not url_has_any_extension(parsed_url, self.deny_extensions) - if allowed and self.canonicalize: - link.url = canonicalize_url(parsed_url) - return allowed - - def matches(self, url): - if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): - return False - if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): - return False - - allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] - denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] - return any(allowed) and not any(denied) diff --git a/scrapy/linkextractor.py b/scrapy/linkextractor.py index 946c82062..d9c85b50a 100644 --- a/scrapy/linkextractor.py +++ b/scrapy/linkextractor.py @@ -2,6 +2,13 @@ Common code and definitions used by Link extractors (located in scrapy.contrib.linkextractor). """ +import re +from urlparse import urlparse + +from scrapy.utils.url import url_is_from_any_domain +from scrapy.utils.url import canonicalize_url, url_is_from_any_domain, url_has_any_extension +from scrapy.utils.misc import arg_to_iter + # common file extensions that are not followed if they occur in links IGNORED_EXTENSIONS = [ @@ -22,3 +29,66 @@ IGNORED_EXTENSIONS = [ # other 'css', 'pdf', 'exe', 'bin', 'rss', 'zip', 'rar', ] + + +_re_type = type(re.compile("", 0)) +_matches = lambda url, regexs: any((r.search(url) for r in regexs)) +_is_valid_url = lambda url: url.split('://', 1)[0] in set(['http', 'https', 'file']) + + +class FilteringLinkExtractor(object): + + def __init__(self, link_extractor, allow, deny, allow_domains, deny_domains, + restrict_xpaths, canonicalize, deny_extensions): + + self.link_extractor = link_extractor + + self.allow_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)] + self.deny_res = [x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)] + + self.allow_domains = set(arg_to_iter(allow_domains)) + self.deny_domains = set(arg_to_iter(deny_domains)) + + self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths)) + self.canonicalize = canonicalize + if deny_extensions is None: + deny_extensions = IGNORED_EXTENSIONS + self.deny_extensions = set(['.' + e for e in arg_to_iter(deny_extensions)]) + + def _link_allowed(self, link): + if not _is_valid_url(link.url): + return False + if self.allow_res and not _matches(link.url, self.allow_res): + return False + if self.deny_res and _matches(link.url, self.deny_res): + return False + parsed_url = urlparse(link.url) + if self.allow_domains and not url_is_from_any_domain(parsed_url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(parsed_url, self.deny_domains): + return False + if self.deny_extensions and url_has_any_extension(parsed_url, self.deny_extensions): + return False + return True + + def matches(self, url): + + if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains): + return False + if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): + return False + + allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] + denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] + return any(allowed) and not any(denied) + + def _process_links(self, links): + links = [x for x in links if self._link_allowed(x)] + if self.canonicalize: + for link in links: + link.url = canonicalize_url(urlparse(link.url)) + links = self.link_extractor._process_links(links) + return links + + def _extract_links(self, *args, **kwargs): + return self.link_extractor._extract_links(*args, **kwargs) diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 72c607096..27d73e271 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -5,6 +5,7 @@ from scrapy.http import HtmlResponse from scrapy.link import Link from scrapy.contrib.linkextractors.htmlparser import HtmlParserLinkExtractor from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor, BaseSgmlLinkExtractor +from scrapy.contrib.linkextractors.lxmlhtml import LxmlLinkExtractor from scrapy.tests import get_testdata @@ -105,19 +106,21 @@ class LinkExtractorTestCase(unittest.TestCase): class SgmlLinkExtractorTestCase(unittest.TestCase): + extractor_cls = SgmlLinkExtractor + def setUp(self): body = get_testdata('link_extractor', 'sgml_linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def test_urls_type(self): '''Test that the resulting urls are regular strings and not a unicode objects''' - lx = SgmlLinkExtractor() + lx = self.extractor_cls() self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response))) def test_extraction(self): '''Test the extractor's behaviour among different situations''' - lx = SgmlLinkExtractor() + lx = self.extractor_cls() self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -126,14 +129,14 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/innertag.html', text=u'inner tag'), ]) - lx = SgmlLinkExtractor(allow=('sample', )) + lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) - lx = SgmlLinkExtractor(allow=('sample', ), unique=False) + lx = self.extractor_cls(allow=('sample', ), unique=False) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -141,20 +144,20 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), ]) - lx = SgmlLinkExtractor(allow=('sample', )) + lx = self.extractor_cls(allow=('sample', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) - lx = SgmlLinkExtractor(allow=('sample', ), deny=('3', )) + lx = self.extractor_cls(allow=('sample', ), deny=('3', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), ]) - lx = SgmlLinkExtractor(allow_domains=('google.com', )) + lx = self.extractor_cls(allow_domains=('google.com', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://www.google.com/something', text=u''), ]) @@ -162,50 +165,78 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): def test_extraction_using_single_values(self): '''Test the extractor's behaviour among different situations''' - lx = SgmlLinkExtractor(allow='sample') + lx = self.extractor_cls(allow='sample') self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), ]) - lx = SgmlLinkExtractor(allow='sample', deny='3') + lx = self.extractor_cls(allow='sample', deny='3') self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), ]) - lx = SgmlLinkExtractor(allow_domains='google.com') + lx = self.extractor_cls(allow_domains='google.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://www.google.com/something', text=u''), ]) - lx = SgmlLinkExtractor(deny_domains='example.com') + lx = self.extractor_cls(deny_domains='example.com') self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://www.google.com/something', text=u''), ]) + def test_nofollow(self): + '''Test the extractor's behaviour for links with rel="nofollow"''' + + html = """Page title<title> + <body> + <div class='links'> + <p><a href="/about.html">About us</a></p> + </div> + <div> + <p><a href="/follow.html">Follow this link</a></p> + </div> + <div> + <p><a href="/nofollow.html" rel="nofollow">Dont follow this one</a></p> + </div> + <div> + <p><a href="/nofollow2.html" rel="blah">Choose to follow or not</a></p> + </div> + </body></html>""" + response = HtmlResponse("http://example.org/somepage/index.html", body=html) + + lx = self.extractor_cls() + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.org/about.html', text=u'About us'), + Link(url='http://example.org/follow.html', text=u'Follow this link'), + Link(url='http://example.org/nofollow.html', text=u'Dont follow this one', nofollow=True), + Link(url='http://example.org/nofollow2.html', text=u'Choose to follow or not'), + ]) + def test_matches(self): url1 = 'http://lotsofstuff.com/stuff1/index' url2 = 'http://evenmorestuff.com/uglystuff/index' - lx = SgmlLinkExtractor(allow=(r'stuff1', )) + lx = self.extractor_cls(allow=(r'stuff1', )) self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), False) - lx = SgmlLinkExtractor(deny=(r'uglystuff', )) + lx = self.extractor_cls(deny=(r'uglystuff', )) self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), False) - lx = SgmlLinkExtractor(allow_domains=('evenmorestuff.com', )) + lx = self.extractor_cls(allow_domains=('evenmorestuff.com', )) self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = SgmlLinkExtractor(deny_domains=('lotsofstuff.com', )) + lx = self.extractor_cls(deny_domains=('lotsofstuff.com', )) self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = SgmlLinkExtractor(allow=('blah1',), deny=('blah2',), + lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), allow_domains=('blah1.com',), deny_domains=('blah2.com',)) self.assertEqual(lx.matches('http://blah1.com/blah1'), True) @@ -214,7 +245,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches('http://blah2.com/blah2'), False) def test_restrict_xpaths(self): - lx = SgmlLinkExtractor(restrict_xpaths=('//div[@id="subwrapper"]', )) + lx = self.extractor_cls(restrict_xpaths=('//div[@id="subwrapper"]', )) self.assertEqual([link for link in lx.extract_links(self.response)], [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -233,7 +264,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='windows-1252') - lx = SgmlLinkExtractor(restrict_xpaths="//div[@class='links']") + lx = self.extractor_cls(restrict_xpaths="//div[@class='links']") self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/about.html', text=u'About us\xa3')]) @@ -248,7 +279,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): """html entities cause SGMLParser to call handle_data hook twice""" body = """<html><body><div><a href="/foo">>\xbe\xa9<\xb6\xab</a></body></html>""" response = HtmlResponse("http://example.org", body=body, encoding='gb18030') - lx = SgmlLinkExtractor(restrict_xpaths="//div") + lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/foo', text=u'>\u4eac<\u4e1c', fragment='', nofollow=False)]) @@ -256,7 +287,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): def test_encoded_url(self): body = """<html><body><div><a href="?page=2">BinB</a></body></html>""" response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') - lx = SgmlLinkExtractor() + lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), ]) @@ -264,7 +295,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): def test_encoded_url_in_restricted_xpath(self): body = """<html><body><div><a href="?page=2">BinB</a></body></html>""" response = HtmlResponse("http://known.fm/AC%2FDC/", body=body, encoding='utf8') - lx = SgmlLinkExtractor(restrict_xpaths="//div") + lx = self.extractor_cls(restrict_xpaths="//div") self.assertEqual(lx.extract_links(response), [ Link(url='http://known.fm/AC%2FDC/?page=2', text=u'BinB', fragment='', nofollow=False), ]) @@ -272,7 +303,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): def test_deny_extensions(self): html = """<a href="page.html">asd</a> and <a href="photo.jpg">""" response = HtmlResponse("http://example.org/", body=html) - lx = SgmlLinkExtractor() + lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ Link(url='http://example.org/page.html', text=u'asd'), ]) @@ -295,7 +326,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): if m: return m.group(1) - lx = SgmlLinkExtractor(process_value=process_value) + lx = self.extractor_cls(process_value=process_value) self.assertEqual(lx.extract_links(response), [Link(url='http://example.org/other/page.html', text='Link text')]) @@ -304,13 +335,12 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): <body><p><a href="item/12.html">Item 12</a></p> </body></html>""" response = HtmlResponse("http://example.org/somepage/index.html", body=html) - lx = SgmlLinkExtractor(restrict_xpaths="//p") + lx = self.extractor_cls(restrict_xpaths="//p") self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) - def test_attrs(self): - lx = SgmlLinkExtractor(attrs="href") + lx = self.extractor_cls(attrs="href") self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -319,7 +349,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/innertag.html', text=u'inner tag'), ]) - lx = SgmlLinkExtractor(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) + lx = self.extractor_cls(attrs=("href","src"), tags=("a","area","img"), deny_extensions=()) self.assertEqual(lx.extract_links(self.response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), @@ -329,7 +359,7 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/innertag.html', text=u'inner tag'), ]) - lx = SgmlLinkExtractor(attrs=None) + lx = self.extractor_cls(attrs=None) self.assertEqual(lx.extract_links(self.response), []) html = """<html><area href="sample1.html"></area><a ref="sample2.html">sample text 2</a></html>""" @@ -339,36 +369,61 @@ class SgmlLinkExtractorTestCase(unittest.TestCase): Link(url='http://example.com/sample1.html', text=u''), ]) - def test_tags(self): html = """<html><area href="sample1.html"></area><a href="sample2.html">sample 2</a><img src="sample2.jpg"/></html>""" response = HtmlResponse("http://example.com/index.html", body=html) - lx = SgmlLinkExtractor(tags=None) + lx = self.extractor_cls(tags=None) self.assertEqual(lx.extract_links(response), []) - lx = SgmlLinkExtractor() + lx = self.extractor_cls() self.assertEqual(lx.extract_links(response), [ Link(url='http://example.com/sample1.html', text=u''), Link(url='http://example.com/sample2.html', text=u'sample 2'), ]) - lx = SgmlLinkExtractor(tags="area") + lx = self.extractor_cls(tags="area") self.assertEqual(lx.extract_links(response), [ Link(url='http://example.com/sample1.html', text=u''), ]) - lx = SgmlLinkExtractor(tags="a") + lx = self.extractor_cls(tags="a") self.assertEqual(lx.extract_links(response), [ Link(url='http://example.com/sample2.html', text=u'sample 2'), ]) - lx = SgmlLinkExtractor(tags=("a","img"), attrs=("href", "src"), deny_extensions=()) + lx = self.extractor_cls(tags=("a","img"), attrs=("href", "src"), deny_extensions=()) self.assertEqual(lx.extract_links(response), [ Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample2.jpg', text=u''), ]) + def test_tags_attrs(self): + html = """ + <html><body> + <div id="item1" data-url="get?id=1"><a href="#">Item 1</a></div> + <div id="item2" data-url="get?id=2"><a href="#">Item 2</a></div> + </body></html> + """ + response = HtmlResponse("http://example.com/index.html", body=html) + + lx = self.extractor_cls(tags='div', attrs='data-url') + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + ]) + + lx = self.extractor_cls(tags=('div',), attrs=('data-url',)) + self.assertEqual(lx.extract_links(response), [ + Link(url='http://example.com/get?id=1', text=u'Item 1', fragment='', nofollow=False), + Link(url='http://example.com/get?id=2', text=u'Item 2', fragment='', nofollow=False) + ]) + + + +class LxmlLinkExtractorTestCase(SgmlLinkExtractorTestCase): + extractor_cls = LxmlLinkExtractor + class HtmlParserLinkExtractorTestCase(unittest.TestCase):