From c1edf0e381e08c13525555653f4287caaec11389 Mon Sep 17 00:00:00 2001
From: samus_ <none@none>
Date: Fri, 2 Jan 2009 14:57:33 +0000
Subject: [PATCH] reverted generator approach because it conflicts with unique
 parameter plus fixed bug in canonicalize routine

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40600
---
 scrapy/trunk/scrapy/link/__init__.py   |  5 ++++-
 scrapy/trunk/scrapy/link/extractors.py | 13 +++++++------
 scrapy/trunk/scrapy/tests/test_link.py |  6 ++----
 3 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py
index c52994995..9d2e06009 100644
--- a/scrapy/trunk/scrapy/link/__init__.py
+++ b/scrapy/trunk/scrapy/link/__init__.py
@@ -42,9 +42,12 @@ class LinkExtractor(FixedSGMLParser):
         self.close()
 
         base_url = self.base_url if self.base_url else response_url
+
+        links = []
         for link in self.links:
             link.url = urljoin(base_url, link.url).strip()
-            yield link
+            links.append(link)
+        return links
 
     def extract_links(self, response):
         # wrapper needed to allow to work directly with text
diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py
index d2a31676c..7b8e7f005 100644
--- a/scrapy/trunk/scrapy/link/extractors.py
+++ b/scrapy/trunk/scrapy/link/extractors.py
@@ -60,19 +60,20 @@ class RegexLinkExtractor(LinkExtractor):
         else:
             links = LinkExtractor.extract_links(self, response)
 
-        links = (link for link in links if _is_valid_url(link.url))
+        links = [link for link in links if _is_valid_url(link.url)]
 
         if self.allow_res:
-            links = (link for link in links if _matches(link.url, self.allow_res))
+            links = [link for link in links if _matches(link.url, self.allow_res)]
         if self.deny_res:
-            links = (link for link in links if not _matches(link.url, self.deny_res))
+            links = [link for link in links if not _matches(link.url, self.deny_res)]
         if self.allow_domains:
-            links = (link for link in links if url_is_from_any_domain(link.url, self.allow_domains))
+            links = [link for link in links if url_is_from_any_domain(link.url, self.allow_domains)]
         if self.deny_domains:
-            links = (link for link in links if not url_is_from_any_domain(link.url, self.deny_domains))
+            links = [link for link in links if not url_is_from_any_domain(link.url, self.deny_domains)]
 
         if self.canonicalize:
-            links = (canonicalize_url(link.url) for link in links)
+            for link in links:
+                link.url = canonicalize_url(link.url)
 
         return links
 
diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py
index 0ebd51733..cbe273d2e 100644
--- a/scrapy/trunk/scrapy/tests/test_link.py
+++ b/scrapy/trunk/scrapy/tests/test_link.py
@@ -18,8 +18,7 @@ class LinkExtractorTestCase(unittest.TestCase):
         response = Response("example.org", "http://example.org/somepage/index.html", body=html)
 
         lx = LinkExtractor()  # default: tag=a, attr=href
-        links = [link for link in lx.extract_links(response)]
-        self.assertEqual(links,
+        self.assertEqual(lx.extract_links(response),
                          [Link(url='http://example.org/somepage/item/12.html', text='Item 12'), 
                           Link(url='http://example.org/about.html', text='About us'),
                           Link(url='http://example.org/othercat.html', text='Other category'), 
@@ -32,8 +31,7 @@ class LinkExtractorTestCase(unittest.TestCase):
         response = Response("example.org", "http://example.org/somepage/index.html", body=html)
 
         lx = LinkExtractor()  # default: tag=a, attr=href
-        links = [link for link in lx.extract_links(response)]
-        self.assertEqual(links,
+        self.assertEqual(lx.extract_links(response),
                          [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
 
     def test_matches(self):