From 8017d85d8a115a3061ef6951cd0989c87d5e7c92 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sat, 1 Feb 2014 22:47:30 +0100 Subject: [PATCH] Fix HtmlParserLinkExtractor and tests after #485 merge --- scrapy/contrib/linkextractors/htmlparser.py | 7 ++++--- scrapy/tests/test_contrib_linkextractors.py | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index aa68ce52f..6e3c5071a 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -62,11 +62,12 @@ class HtmlParserLinkExtractor(HTMLParser): self.current_link = link def handle_endtag(self, tag): - self.current_link = None + if self.scan_tag(tag): + self.current_link = None def handle_data(self, data): - if self.current_link and not self.current_link.text: - self.current_link.text = data.strip() + if self.current_link: + self.current_link.text = self.current_link.text + data def matches(self, url): """This extractor matches with any url, since diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 20a0dffde..de05cbe98 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -317,7 +317,8 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase): [Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'), - Link(url='http://www.google.com/something', text=u''),]) + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'),]) class RegexLinkExtractorTestCase(unittest.TestCase): @@ -332,7 +333,8 @@ class RegexLinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(self.response), [Link(url='http://example.com/sample2.html', text=u'sample 2'), Link(url='http://example.com/sample3.html', text=u'sample 3 text'), - Link(url='http://www.google.com/something', text=u''),]) + Link(url='http://www.google.com/something', text=u''), + Link(url='http://example.com/innertag.html', text=u'inner tag'),]) if __name__ == "__main__":