diff --git a/scrapy/trunk/scrapy/link/__init__.py b/scrapy/trunk/scrapy/link/__init__.py index 9d2e06009..8bc55d91f 100644 --- a/scrapy/trunk/scrapy/link/__init__.py +++ b/scrapy/trunk/scrapy/link/__init__.py @@ -3,21 +3,22 @@ LinkExtractor provides en efficient way to extract links from pages """ from scrapy.utils.python import FixedSGMLParser -from scrapy.utils.url import urljoin_rfc as urljoin +from scrapy.utils.url import safe_url_string, urljoin_rfc as urljoin class LinkExtractor(FixedSGMLParser): """LinkExtractor are used to extract links from web pages. They are instantiated and later "applied" to a Response using the extract_links - method which must receive a Response object and return a dict whoose keys - are the (absolute) urls to follow, and its values any arbitrary data. In - this case the values are the text of the hyperlink. + method which must receive a Response object and return a list of Link objects + containing the (absolute) urls to follow, and the links texts. This is the base LinkExtractor class that provides enough basic functionality for extracting links to follow, but you could override this class or create a new one if you need some additional functionality. The only requisite is that the new (or overrided) class must provide a - extract_links method that receives a Response and returns a dict with the - links to follow as its keys. + extract_links method that receives a Response and returns a list of Link objects. + + This LinkExtractor always returns percent-encoded URLs, using the detected encoding + from the response. The constructor arguments are: @@ -36,22 +37,38 @@ class LinkExtractor(FixedSGMLParser): self.current_link = None self.unique = unique - def _extract_links(self, response_text, response_url): + def _extract_links(self, response_text, response_url, response_encoding): self.reset() self.feed(response_text) self.close() - base_url = self.base_url if self.base_url else response_url + links = self.links + if self.unique: + seen = set() + def _seen(url): + if url in seen: + return True + else: + seen.add(url) + return False - links = [] - for link in self.links: + links = [link for link in links if not _seen(link.url)] + + ret = [] + base_url = self.base_url if self.base_url else response_url + for link in links: link.url = urljoin(base_url, link.url).strip() - links.append(link) - return links + link.url = safe_url_string(link.url, response_encoding) + link.text = link.text.decode(response_encoding) + ret.append(link) + + return ret def extract_links(self, response): # wrapper needed to allow to work directly with text - return self._extract_links(response.body.to_string(), response.url) + return self._extract_links(response.body.to_string(), + response.url, + response.body.get_real_encoding()) def reset(self): FixedSGMLParser.reset(self) @@ -64,10 +81,9 @@ class LinkExtractor(FixedSGMLParser): if self.scan_tag(tag): for attr, value in attrs: if self.scan_attr(attr): - if not self.unique or not value in [link.url for link in self.links]: - link = Link(url=value) - self.links.append(link) - self.current_link = link + link = Link(url=value) + self.links.append(link) + self.current_link = link def unknown_endtag(self, tag): self.current_link = None diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 7b8e7f005..8ee90f86f 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -47,10 +47,9 @@ class RegexLinkExtractor(LinkExtractor): self.deny_domains = set(deny_domains) self.restrict_xpaths = restrict_xpaths self.canonicalize = canonicalize - self.unique = unique tag_func = lambda x: x in tags attr_func = lambda x: x in attrs - LinkExtractor.__init__(self, tag=tag_func, attr=attr_func) + LinkExtractor.__init__(self, tag=tag_func, attr=attr_func, unique=unique) def extract_links(self, response): if self.restrict_xpaths: diff --git a/scrapy/trunk/scrapy/tests/sample_data/image_linkextractor.html b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html similarity index 77% rename from scrapy/trunk/scrapy/tests/sample_data/image_linkextractor.html rename to scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html index 750fbafd3..722240c03 100644 --- a/scrapy/trunk/scrapy/tests/sample_data/image_linkextractor.html +++ b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/image_linkextractor.html @@ -1,7 +1,7 @@ - -Sample page with image links for testing ImageLinkExtractor + +Sample page with image links for testing HTMLImageLinkExtractor
diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html new file mode 100644 index 000000000..68609d8d3 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_latin1.html @@ -0,0 +1,15 @@ + + + + +Sample page with links for testing RegexLinkExtractor + + +
+
+ +
+sample á text +
+ + diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html new file mode 100644 index 000000000..f9166adbe --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/linkextractor_noenc.html @@ -0,0 +1,14 @@ + + + +Sample page without encoding for testing LinkExtractor + + +
+
+ +
+sample € text +
+ + diff --git a/scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html new file mode 100644 index 000000000..e0262fefc --- /dev/null +++ b/scrapy/trunk/scrapy/tests/sample_data/link_extractor/regex_linkextractor.html @@ -0,0 +1,17 @@ + + + +Sample page with links for testing RegexLinkExtractor + + +
+
+sample 1 + +
+sample 3 text +sample 3 repetition + +
+ + diff --git a/scrapy/trunk/scrapy/tests/test_link.py b/scrapy/trunk/scrapy/tests/test_link.py index cbe273d2e..a5a40436e 100644 --- a/scrapy/trunk/scrapy/tests/test_link.py +++ b/scrapy/trunk/scrapy/tests/test_link.py @@ -34,6 +34,27 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + def test_extraction_encoding(self): + base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') + body = open(os.path.join(base_path, 'linkextractor_noenc.html'), 'r').read() + response_utf8 = Response(url='http://example.com/utf8', domain='example.com', body=ResponseBody(body), headers={'Content-Type': ['text/html; charset=utf-8']}) + response_noenc = Response(url='http://example.com/noenc', domain='example.com', body=ResponseBody(body)) + body = open(os.path.join(base_path, 'linkextractor_latin1.html'), 'r').read() + response_latin1 = Response(url='http://example.com/latin1', domain='example.com', body=ResponseBody(body)) + + lx = LinkExtractor() + self.assertEqual(lx.extract_links(response_utf8), + [ Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ]) + + self.assertEqual(lx.extract_links(response_noenc), + [ Link(url='http://example.com/sample_%C3%B1.html', text=''), + Link(url='http://example.com/sample_%E2%82%AC.html', text='sample \xe2\x82\xac text'.decode('utf-8')) ]) + + self.assertEqual(lx.extract_links(response_latin1), + [ Link(url='http://example.com/sample_%F1.html', text=''), + Link(url='http://example.com/sample_%E1.html', text='sample \xe1 text'.decode('latin1')) ]) + def test_matches(self): url1 = 'http://lotsofstuff.com/stuff1/index' url2 = 'http://evenmorestuff.com/uglystuff/index' @@ -42,6 +63,57 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), True) +class RegexLinkExtractorTestCase(unittest.TestCase): + def setUp(self): + base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') + body = open(os.path.join(base_path, 'regex_linkextractor.html'), 'r').read() + self.response = Response(url='http://example.com/index', domain='example.com', body=ResponseBody(body)) + + def test_urls_type(self): + '''Test that the resulting urls are regular strings and not a unicode objects''' + lx = RegexLinkExtractor() + self.assertTrue(all(isinstance(link.url, str) for link in lx.extract_links(self.response))) + + def test_extraction(self): + '''Test the extractor's behaviour among different situations''' + + lx = RegexLinkExtractor() + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text='sample 1'), + Link(url='http://example.com/sample2.html', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://www.google.com/something', text='') ]) + + lx = RegexLinkExtractor(allow=('sample', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text='sample 1'), + Link(url='http://example.com/sample2.html', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text') ]) + + lx = RegexLinkExtractor(allow=('sample', ), unique=False) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text='sample 1'), + Link(url='http://example.com/sample2.html', text=''), + Link(url='http://example.com/sample3.html', text='sample 3 text'), + Link(url='http://example.com/sample3.html', text='sample 3 repetition') ]) + + lx = RegexLinkExtractor(allow=('sample', ), deny=('3', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample1.html', text='sample 1'), + Link(url='http://example.com/sample2.html', text='') ]) + + lx = RegexLinkExtractor(allow_domains=('google.com', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://www.google.com/something', text='') ]) + + lx = RegexLinkExtractor(tags=('img', ), attrs=('src', )) + self.assertEqual([link for link in lx.extract_links(self.response)], + [ Link(url='http://example.com/sample2.jpg', text='') ]) + + def test_matches(self): + url1 = 'http://lotsofstuff.com/stuff1/index' + url2 = 'http://evenmorestuff.com/uglystuff/index' + lx = RegexLinkExtractor(allow=(r'stuff1', )) self.assertEqual(lx.matches(url1), True) self.assertEqual(lx.matches(url2), False) @@ -67,8 +139,9 @@ class LinkExtractorTestCase(unittest.TestCase): #class HTMLImageLinkExtractorTestCase(unittest.TestCase): # def setUp(self): -# body = open(os.path.join(os.path.dirname(__file__), 'sample_data/image_linkextractor.html'), 'r').read() -# self.response = Response(url='http://examplesite.com/index', domain='examplesite.com', body=ResponseBody(body)) +# base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') +# body = open(os.path.join(base_path 'image_linkextractor.html'), 'r').read() +# self.response = Response(url='http://example.com/index', domain='example.com', body=ResponseBody(body)) # def test_urls_type(self): # '''Test that the resulting urls are regular strings and not a unicode objects''' @@ -82,30 +155,30 @@ class LinkExtractorTestCase(unittest.TestCase): # lx = HTMLImageLinkExtractor(locations=('//img', )) # links_1 = lx.extract_links(self.response) # self.assertEqual(links_1, -# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), -# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ]) +# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), +# Link(url='http://example.com/sample2.jpg', text=u'sample 2'), +# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) # lx = HTMLImageLinkExtractor(locations=('//img', ), unique=False) # links_2 = lx.extract_links(self.response, unique=False) # self.assertEqual(links_2, -# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), -# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4'), -# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4 repetition') ]) +# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), +# Link(url='http://example.com/sample2.jpg', text=u'sample 2'), +# Link(url='http://example.com/sample4.jpg', text=u'sample 4'), +# Link(url='http://example.com/sample4.jpg', text=u'sample 4 repetition') ]) # lx = HTMLImageLinkExtractor(locations=('//div[@id="wrapper"]', ) # links_3 = lx.extract_links(self.response) # self.assertEqual(links_3, -# [ Link(url='http://examplesite.com/sample1.jpg', text=u'sample 1'), -# Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://examplesite.com/sample4.jpg', text=u'sample 4') ]) +# [ Link(url='http://example.com/sample1.jpg', text=u'sample 1'), +# Link(url='http://example.com/sample2.jpg', text=u'sample 2'), +# Link(url='http://example.com/sample4.jpg', text=u'sample 4') ]) # lx = HTMLImageLinkExtractor(locations=('//a', ) # links_4 = lx.extract_links(self.response) # self.assertEqual(links_4, -# [ Link(url='http://examplesite.com/sample2.jpg', text=u'sample 2'), -# Link(url='http://examplesite.com/sample3.html', text=u'sample 3') ]) +# [ Link(url='http://example.com/sample2.jpg', text=u'sample 2'), +# Link(url='http://example.com/sample3.html', text=u'sample 3') ]) if __name__ == "__main__": unittest.main()