RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

handle case when attributes are not separated by space (still recognizable because of quotes)

This commit is contained in:
Martin Olveyra 2011-01-24 18:40:42 -02:00
parent 18e097d496
commit 32adbea545
3 changed files with 28 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/contrib/ibl/htmlpage.py
View File

@ -80,8 +80,8 @@ class HtmlTag(HtmlDataFragment):
def __repr__(self):
return str(self)
_ATTR = "((?:[^=/>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?"
_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>?"
_ATTR = "((?:[^=/<>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?"
_TAG = "<(\/?)(\w+(?::\w+)?)((?:\s*" + _ATTR + ")+\s*|\s*)(\/?)>?"
_DOCTYPE = r"<!DOCTYPE.*?>"
_SCRIPT = "(<script.*?>)(.*?)(</script.*?>)"
_COMMENT = "(<!--.*?-->)"

6
scrapy/tests/test_contrib_ibl/test_htmlpage.py
View File

@ -147,3 +147,9 @@ class TestParseHtml(TestCase):
"""Test parsing of some malformed cases"""
parsed = [_decode_element(d) for d in PARSED8]
self._test_sample(PAGE8, parsed)
def test_malformed2(self):
"""Test case when attributes are not separated by space (still recognizable because of quotes)"""
parsed = [_decode_element(d) for d in PARSED9]
self._test_sample(PAGE9, parsed)

20
scrapy/tests/test_contrib_ibl/test_htmlpage_data.py
View File

@ -255,3 +255,23 @@ PARSED8 = [
{'attributes' : {}, 'end': 103, 'start': 98, 'tag': u'td', 'tag_type': 2},
{'attributes' : {u'width': u'5'}, 'end': 120, 'start': 103, 'tag': u'table', 'tag_type': 1}
]
PAGE9 = u"""\
<html>\
<body>\
<img width='230' height='150'src='/images/9589.jpg' >\
<a href="/product/9589">Click here</a>\
</body>\
</html>\
"""
PARSED9 = [
{'attributes' : {}, 'end': 6, 'start': 0, 'tag': 'html', 'tag_type': 1},
{'attributes' : {}, 'end': 12, 'start': 6, 'tag': 'body', 'tag_type': 1},
{'attributes' : {'width': '230', 'height': '150', 'src': '/images/9589.jpg'}, 'end': 65, 'start': 12, 'tag': 'img', 'tag_type': 1},
{'attributes' : {'href': '/product/9589'}, 'end': 89, 'start': 65, 'tag': 'a', 'tag_type': 1},
{'end': 99, 'start': 89},
{'attributes' : {}, 'end': 103, 'start': 99, 'tag': 'a', 'tag_type': 2},
{'attributes' : {}, 'end': 110, 'start': 103, 'tag': 'body', 'tag_type': 2},
{'attributes' : {}, 'end': 117, 'start': 110, 'tag': 'html', 'tag_type': 2},
]