RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix HtmlParserLinkExtractor and tests after #485 merge

This commit is contained in:
Paul Tremberth 2014-02-01 22:47:30 +01:00
parent e31fb49320
commit 8017d85d8a
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/contrib/linkextractors/htmlparser.py
View File

@ -62,11 +62,12 @@ class HtmlParserLinkExtractor(HTMLParser):
self.current_link = link
def handle_endtag(self, tag):
self.current_link = None
if self.scan_tag(tag):
self.current_link = None
def handle_data(self, data):
if self.current_link and not self.current_link.text:
self.current_link.text = data.strip()
if self.current_link:
self.current_link.text = self.current_link.text + data
def matches(self, url):
"""This extractor matches with any url, since

6
scrapy/tests/test_contrib_linkextractors.py
View File

@ -317,7 +317,8 @@ class HtmlParserLinkExtractorTestCase(unittest.TestCase):
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://example.com/sample3.html', text=u'sample 3 repetition'),
Link(url='http://www.google.com/something', text=u''),])
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
class RegexLinkExtractorTestCase(unittest.TestCase):
@ -332,7 +333,8 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.extract_links(self.response),
[Link(url='http://example.com/sample2.html', text=u'sample 2'),
Link(url='http://example.com/sample3.html', text=u'sample 3 text'),
Link(url='http://www.google.com/something', text=u''),])
Link(url='http://www.google.com/something', text=u''),
Link(url='http://example.com/innertag.html', text=u'inner tag'),])
if __name__ == "__main__":