RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

reverted generator approach because it conflicts with unique parameter plus fixed bug in canonicalize routine 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40600
This commit is contained in:
samus_ 2009-01-02 14:57:33 +00:00
parent e5a764cd2a
commit c1edf0e381
3 changed files with 13 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/trunk/scrapy/link/__init__.py
View File

@ -42,9 +42,12 @@ class LinkExtractor(FixedSGMLParser):
self.close()
base_url = self.base_url if self.base_url else response_url
links = []
for link in self.links:
link.url = urljoin(base_url, link.url).strip()
yield link
links.append(link)
return links
def extract_links(self, response):
# wrapper needed to allow to work directly with text

13
scrapy/trunk/scrapy/link/extractors.py
View File

@ -60,19 +60,20 @@ class RegexLinkExtractor(LinkExtractor):
else:
links = LinkExtractor.extract_links(self, response)
links = (link for link in links if _is_valid_url(link.url))
links = [link for link in links if _is_valid_url(link.url)]
if self.allow_res:
links = (link for link in links if _matches(link.url, self.allow_res))
links = [link for link in links if _matches(link.url, self.allow_res)]
if self.deny_res:
links = (link for link in links if not _matches(link.url, self.deny_res))
links = [link for link in links if not _matches(link.url, self.deny_res)]
if self.allow_domains:
links = (link for link in links if url_is_from_any_domain(link.url, self.allow_domains))
links = [link for link in links if url_is_from_any_domain(link.url, self.allow_domains)]
if self.deny_domains:
links = (link for link in links if not url_is_from_any_domain(link.url, self.deny_domains))
links = [link for link in links if not url_is_from_any_domain(link.url, self.deny_domains)]
if self.canonicalize:
links = (canonicalize_url(link.url) for link in links)
for link in links:
link.url = canonicalize_url(link.url)
return links

6
scrapy/trunk/scrapy/tests/test_link.py
View File

@ -18,8 +18,7 @@ class LinkExtractorTestCase(unittest.TestCase):
response = Response("example.org", "http://example.org/somepage/index.html", body=html)
lx = LinkExtractor() # default: tag=a, attr=href
links = [link for link in lx.extract_links(response)]
self.assertEqual(links,
self.assertEqual(lx.extract_links(response),
[Link(url='http://example.org/somepage/item/12.html', text='Item 12'),
Link(url='http://example.org/about.html', text='About us'),
Link(url='http://example.org/othercat.html', text='Other category'),
@ -32,8 +31,7 @@ class LinkExtractorTestCase(unittest.TestCase):
response = Response("example.org", "http://example.org/somepage/index.html", body=html)
lx = LinkExtractor() # default: tag=a, attr=href
links = [link for link in lx.extract_links(response)]
self.assertEqual(links,
self.assertEqual(lx.extract_links(response),
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
def test_matches(self):