diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index 069d22d19..e80b20122 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -33,22 +33,24 @@ class LinkExtractor(FixedSGMLParser): FixedSGMLParser.__init__(self) self.scan_tag = tag if callable(tag) else lambda t: t == tag self.scan_attr = attr if callable(attr) else lambda a: a == attr - self.inside_link = False + self.current_link = None - def extract_urls(self, response): + def extract_urls(self, response, unique=False): self.reset() + self.unique = unique self.feed(response.body.to_string()) self.close() base_url = self.base_url if self.base_url else response.url - urls = {} - for link, text in self.links.iteritems(): - urls[urljoin(base_url, link)] = text - return urls + ret = [] + for link in self.links: + link.url = urljoin(base_url, link.url) + ret.append(link) + return ret def reset(self): FixedSGMLParser.reset(self) - self.links = {} + self.links = [] self.base_url = None def unknown_starttag(self, tag, attrs): @@ -57,12 +59,27 @@ class LinkExtractor(FixedSGMLParser): if self.scan_tag(tag): for attr, value in attrs: if self.scan_attr(attr): - self.links[value] = "" - self.inside_link = value + if not self.unique or not value in [link.url for link in self.links]: + link = Link(url=value) + self.links.append(link) + self.current_link = link def unknown_endtag(self, tag): - self.inside_link = False + self.current_link = None def handle_data(self, data): - if self.inside_link and not self.links.get(self.inside_link, None): - self.links[self.inside_link] = data + if self.current_link and not self.current_link.text: + self.current_link.text = data + + +class Link(object): + """ + Link objects represent an extracted link by the LinkExtractor. + At the moment, it contains just the url and link text. + """ + def __init__(self, url, text=''): + self.url = url + self.text = text + + def __eq__(self, other): + return self.url == other.url and self.text == other.text diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 2153da7e1..dc9bb745a 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -50,27 +50,27 @@ class RegexLinkExtractor(LinkExtractor): attr_func = lambda x: x in attrs LinkExtractor.__init__(self, tag=tag_func, attr=attr_func) - def extract_urls(self, response): + def extract_urls(self, response, unique=True): if self.restrict_xpaths: response = new_response_from_xpaths(response, self.restrict_xpaths) - url_text = LinkExtractor.extract_urls(self, response) - urls = [u for u in url_text.iterkeys() if _is_valid_url(u)] + links = LinkExtractor.extract_urls(self, response, unique) + links = [link for link in links if _is_valid_url(link.url)] if self.allow_res: - urls = [u for u in urls if _matches(u, self.allow_res)] + links = [link for link in links if _matches(link.url, self.allow_res)] if self.deny_res: - urls = [u for u in urls if not _matches(u, self.deny_res)] + links = [link for link in links if not _matches(link.url, self.deny_res)] if self.allow_domains: - urls = [u for u in urls if url_is_from_any_domain(u, self.allow_domains)] + links = [link for link in links if url_is_from_any_domain(link.url, self.allow_domains)] if self.deny_domains: - urls = [u for u in urls if not url_is_from_any_domain(u, self.deny_domains)] - - res = {} + links = [link for link in links if not url_is_from_any_domain(link.url, self.deny_domains)] + if self.canonicalize: - for u in urls: - res[canonicalize_url(u)] = url_text[u] - else: - for u in urls: - res[u] = url_text[u] - return res + for link in links: + link.url = canonicalize_url(link.url) + + return links + + def match(self, url): + return any(regex.search(url) for regex in self.allow_res) and not any(regex.search(url) for regex in self.deny_res) diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index e4cf3ee58..8c6b603d5 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Response -from scrapy.link import LinkExtractor +from scrapy.link import LinkExtractor, Link class LinkExtractorTestCase(unittest.TestCase): def test_basic(self): @@ -16,10 +16,10 @@ class LinkExtractorTestCase(unittest.TestCase): lx = LinkExtractor() # default: tag=a, attr=href self.assertEqual(lx.extract_urls(response), - {'http://example.org/somepage/item/12.html': 'Item 12', - 'http://example.org/about.html': 'About us', - 'http://example.org/othercat.html': 'Other category', - 'http://example.org/': ''}) + [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), + Link(url='http://example.org/about.html', text='About us'), + Link(url='http://example.org/othercat.html', text='Other category'), + Link(url='http://example.org/', text='')]) def test_base_url(self): html = """Page title<title><base href="http://otherdomain.com/base/" /> @@ -29,7 +29,7 @@ class LinkExtractorTestCase(unittest.TestCase): lx = LinkExtractor() # default: tag=a, attr=href self.assertEqual(lx.extract_urls(response), - {'http://otherdomain.com/base/item/12.html': 'Item 12'}) + [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) if __name__ == "__main__": unittest.main()