Added some improvements to LinkExtractor

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40353

scrapy/trunk/scrapy/link/__init__.py

@@ -33,22 +33,24 @@ class LinkExtractor(FixedSGMLParser):
         FixedSGMLParser.__init__(self)
         self.scan_tag = tag if callable(tag) else lambda t: t == tag
         self.scan_attr = attr if callable(attr) else lambda a: a == attr
-        self.inside_link = False
+        self.current_link = None
 
-    def extract_urls(self, response):
+    def extract_urls(self, response, unique=False):
         self.reset()
+        self.unique = unique
         self.feed(response.body.to_string())
         self.close()
         
         base_url = self.base_url if self.base_url else response.url
-        urls = {}
-        for link, text in self.links.iteritems():
-            urls[urljoin(base_url, link)] = text
-        return urls
+        ret = []
+        for link in self.links:
+            link.url = urljoin(base_url, link.url)
+            ret.append(link)
+        return ret
 
     def reset(self):
         FixedSGMLParser.reset(self)
-        self.links = {}
+        self.links = []
         self.base_url = None
 
     def unknown_starttag(self, tag, attrs):
@@ -57,12 +59,27 @@ class LinkExtractor(FixedSGMLParser):
         if self.scan_tag(tag):
             for attr, value in attrs:
                 if self.scan_attr(attr):
-                    self.links[value] = ""
-                    self.inside_link = value
+                    if not self.unique or not value in [link.url for link in self.links]:
+                        link = Link(url=value)
+                        self.links.append(link)
+                        self.current_link = link
 
     def unknown_endtag(self, tag):
-        self.inside_link = False
+        self.current_link = None
 
     def handle_data(self, data):
-        if self.inside_link and not self.links.get(self.inside_link, None):
-            self.links[self.inside_link] = data
+        if self.current_link and not self.current_link.text:
+            self.current_link.text = data
+
+
+class Link(object):
+    """
+    Link objects represent an extracted link by the LinkExtractor.
+    At the moment, it contains just the url and link text.
+    """
+    def __init__(self, url, text=''):
+        self.url = url
+        self.text = text
+
+    def __eq__(self, other):
+        return self.url == other.url and self.text == other.text

scrapy/trunk/scrapy/link/extractors.py

@@ -50,27 +50,27 @@ class RegexLinkExtractor(LinkExtractor):
         attr_func = lambda x: x in attrs
         LinkExtractor.__init__(self, tag=tag_func, attr=attr_func)
 
-    def extract_urls(self, response):
+    def extract_urls(self, response, unique=True):
         if self.restrict_xpaths:
             response = new_response_from_xpaths(response, self.restrict_xpaths)
 
-        url_text = LinkExtractor.extract_urls(self, response)
-        urls = [u for u in url_text.iterkeys() if _is_valid_url(u)]
+        links = LinkExtractor.extract_urls(self, response, unique)
+        links = [link for link in links if _is_valid_url(link.url)]
 
         if self.allow_res:
-            urls = [u for u in urls if _matches(u, self.allow_res)]
+            links = [link for link in links if _matches(link.url, self.allow_res)]
         if self.deny_res:
-            urls = [u for u in urls if not _matches(u, self.deny_res)]
+            links = [link for link in links if not _matches(link.url, self.deny_res)]
         if self.allow_domains:
-            urls = [u for u in urls if url_is_from_any_domain(u, self.allow_domains)]
+            links = [link for link in links if url_is_from_any_domain(link.url, self.allow_domains)]
         if self.deny_domains:
-            urls = [u for u in urls if not url_is_from_any_domain(u, self.deny_domains)]
-
-        res = {}
+            links = [link for link in links if not url_is_from_any_domain(link.url, self.deny_domains)]
+        
         if self.canonicalize:
-            for u in urls:
-                res[canonicalize_url(u)] = url_text[u]
-        else:
-            for u in urls:
-                res[u] = url_text[u]
-        return res
+            for link in links:
+                link.url = canonicalize_url(link.url)
+
+        return links
+
+    def match(self, url):
+        return any(regex.search(url) for regex in self.allow_res) and not any(regex.search(url) for regex in self.deny_res)

scrapy/trunk/scrapy/tests/test_link.py

@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.http import Response
-from scrapy.link import LinkExtractor
+from scrapy.link import LinkExtractor, Link
 
 class LinkExtractorTestCase(unittest.TestCase):
     def test_basic(self):
@@ -16,10 +16,10 @@ class LinkExtractorTestCase(unittest.TestCase):
 
         lx = LinkExtractor()  # default: tag=a, attr=href
         self.assertEqual(lx.extract_urls(response), 
-                         {'http://example.org/somepage/item/12.html': 'Item 12', 
-                          'http://example.org/about.html': 'About us', 
-                          'http://example.org/othercat.html': 'Other category', 
-                          'http://example.org/': ''})
+                         [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), 
+                          Link(url='http://example.org/about.html', text='About us'),
+                          Link(url='http://example.org/othercat.html', text='Other category'), 
+                          Link(url='http://example.org/', text='')])
 
     def test_base_url(self):
         html = """<html><head><title>Page title<title><base href="http://otherdomain.com/base/" />
@@ -29,7 +29,7 @@ class LinkExtractorTestCase(unittest.TestCase):
 
         lx = LinkExtractor()  # default: tag=a, attr=href
         self.assertEqual(lx.extract_urls(response), 
-                         {'http://otherdomain.com/base/item/12.html': 'Item 12'})
+                         [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
 
 if __name__ == "__main__":
     unittest.main()