Simplification of html parsing algorithm, fixed some tests (with new algorithm, comments inside bigger text region are generated separated from the text). Added test for a case not correctly handled by previous algorithm. Fixed test checking

scrapy/contrib/ibl/htmlpage.py

@@ -83,93 +83,18 @@ class HtmlTag(HtmlDataFragment):
 _ATTR = "((?:[^=/>\s]|/(?!>))+)(?:\s*=(?:\s*\"(.*?)\"|\s*'(.*?)'|([^>\s]+))?)?"
 _TAG = "<(\/?)(\w+(?::\w+)?)((?:\s+" + _ATTR + ")+\s*|\s*)(\/?)>"
 _DOCTYPE = r"<!DOCTYPE.*?>"
+_SCRIPT = "(<script.*?>)(.*?)(</script.*?>)"
+_COMMENT = "(<!--.*?-->)"
 
 _ATTR_REGEXP = re.compile(_ATTR, re.I | re.DOTALL)
-_HTML_REGEXP = re.compile(_TAG, re.I | re.DOTALL)
+_HTML_REGEXP = re.compile("%s|%s|%s" % (_COMMENT, _SCRIPT, _TAG), re.I | re.DOTALL)
 _DOCTYPE_REGEXP = re.compile("(?:%s)" % _DOCTYPE)
-_COMMENT_RE = re.compile("(<!--.*?-->)", re.DOTALL)
-_SCRIPT_RE = re.compile("(<script.*?>).*?(</script.*?>)", re.DOTALL | re.I)
+_COMMENT_REGEXP = re.compile(_COMMENT, re.DOTALL)
 
 def parse_html(text):
     """Higher level html parser. Calls lower level parsers and joins sucesive
     HtmlDataFragment elements in a single one.
     """
-    script_layer = lambda x: _parse_clean_html(x, _SCRIPT_RE, HtmlTag, _simple_parse_html)
-    comment_layer = lambda x: _parse_clean_html(x, _COMMENT_RE, HtmlDataFragment, script_layer)
-    delayed_element = None
-    for element in comment_layer(text):
-        if isinstance(element, HtmlTag):
-            if delayed_element is not None:
-                yield delayed_element
-                delayed_element = None
-            yield element
-        else:# element is HtmlDataFragment
-            if delayed_element is not None:
-                delayed_element.start = min(element.start, delayed_element.start)
-                delayed_element.end = max(element.end, delayed_element.end)
-            else:
-                delayed_element = element
-    if delayed_element is not None:
-        yield delayed_element
-
-def _parse_clean_html(text, regex, htype, func):
-    """
-    Removes regions from text, passes the cleaned text to the lower parse layer,
-    and reinserts removed regions.
-    regex - regular expression that defines regions to be removed/re inserted
-    htype - the html parser type of the removed elements
-    func - function that performs the lower parse layer
-    """
-    removed = [[m.start(), m.end(), m.groups()] for m in regex.finditer(text)]
-    
-    cleaned = regex.sub("", text)
-    shift = 0
-    for element in func(cleaned):
-        element.start += shift
-        element.end += shift
-        while removed:
-            if element.end <= removed[0][0]:
-                yield element
-                break
-            else:
-                start, end, groups = removed.pop(0)
-                add = end - start
-                element.end += add
-                shift += add
-                if element.start >= start:
-                    element.start += add
-                elif isinstance(element, HtmlTag):
-                    yield element
-                    break
-                
-                if element.start < start:
-                    yield HtmlDataFragment(element.start, start)
-                    element.start = end
-                
-                if htype == HtmlTag:
-                    begintag = _parse_tag(_HTML_REGEXP.match(groups[0]))
-                    endtag = _parse_tag(_HTML_REGEXP.match(groups[1]))
-                    begintag.start = start
-                    begintag.end += start
-                    
-                    endtag.start = end - endtag.end
-                    endtag.end = end
-                    content = None
-                    if begintag.end < endtag.start:
-                        content = HtmlDataFragment(begintag.end, endtag.start)
-                    yield begintag
-                    if content is not None:
-                        yield content
-                    yield endtag
-                else:
-                    yield htype(start, end)
-        else:
-            yield element
-
-def _simple_parse_html(text):
-    """Simple html parse. It returns a sequence of HtmlTag and HtmlDataFragment
-    objects. Does not ignore any region.
-    """
     # If have doctype remove it.
     start_pos = 0
     match = _DOCTYPE_REGEXP.match(text)
@@ -182,19 +107,49 @@ def _simple_parse_html(text):
             
         if start > prev_end:
             yield HtmlDataFragment(prev_end, start)
-        
-        yield _parse_tag(match)
+
+        if match.groups()[0] is not None: # comment
+            yield HtmlDataFragment(start, end)
+        elif match.groups()[1] is not None: # <script>...</script>
+            for e in _parse_script(match):
+                yield e
+        else: # tag
+            yield _parse_tag(match)
         prev_end = end
     textlen = len(text)
     if prev_end < textlen:
         yield HtmlDataFragment(prev_end, textlen)
 
+def _parse_script(match):
+    """parse a <script>...</script> region matched by _HTML_REGEXP"""
+    open_text, content, close_text = match.groups()[1:4]
+
+    open_tag = _parse_tag(_HTML_REGEXP.match(open_text))
+    open_tag.start = match.start()
+    open_tag.end = match.start() + len(open_text)
+
+    close_tag = _parse_tag(_HTML_REGEXP.match(close_text))
+    close_tag.start = match.end() - len(close_text)
+    close_tag.end = match.end()
+    
+    yield open_tag
+    if open_tag.end < close_tag.start:
+        start_pos = 0
+        for m in _COMMENT_REGEXP.finditer(content):
+            if m.start() > start_pos:
+                yield HtmlDataFragment(open_tag.end + start_pos, open_tag.end + m.start())
+            yield HtmlDataFragment(open_tag.end + m.start(), open_tag.end + m.end())
+            start_pos = m.end()
+        if open_tag.end + start_pos < close_tag.start:
+            yield HtmlDataFragment(open_tag.end + start_pos, close_tag.start)
+    yield close_tag
+
 def _parse_tag(match):
     """
     parse a tag matched by _HTML_REGEXP
     """
     data = match.groups()
-    closing, tag, attr_text = data[:3]
+    closing, tag, attr_text = data[4:7]
     # if tag is None then the match is a comment
     if tag is not None:
         unpaired = data[-1]

scrapy/tests/test_contrib_ibl/samples/samples_htmlpage_0.json

@@ -208,11 +208,19 @@
                 "end": 1073, 
                 "start": 1043, 
                 "tag_type": 1
-        }, 
+        },
+        {       
+                "start": 1073,
+                "end": 1074
+        },
         {
-                "start": 1073, 
+                "start": 1074, 
+                "end": 2052
+        },
+        {
+                "start": 2052,
                 "end": 2053
-        }, 
+        },
         {
                 "attributes": {}, 
                 "tag": "script", 
@@ -3088,4 +3096,4 @@
                 "start": 13843, 
                 "tag_type": 2
         }
-]
+]

scrapy/tests/test_contrib_ibl/samples/samples_htmlpage_1.json

@@ -208,8 +208,16 @@
         }, 
         {
                 "start": 1954, 
+                "end": 1956
+        },
+        {
+                "start": 1956,
+                "end": 1979
+        },
+        {
+                "start": 1979,
                 "end": 1980
-        }, 
+        },
         {
                 "attributes": {
                         "src": "http://images.play.com/sitetrak/cmdatatagutilsA.js", 
@@ -266,6 +274,14 @@
         }, 
         {
                 "start": 2282, 
+                "end": 2283
+        },
+        {
+                "start": 2283,
+                "end": 2437
+        },
+        {
+                "start": 2437,
                 "end": 2438
         }, 
         {
@@ -325,6 +341,14 @@
         }, 
         {
                 "start": 2860, 
+                "end": 2861
+        },
+        {
+                "start": 2861,
+                "end": 2882
+        },
+        {
+                "start": 2882,
                 "end": 2884
         }, 
         {
@@ -21941,4 +21965,4 @@
                 "tag_type": 2, 
                 "start": 72792
         }
-]
+]

scrapy/tests/test_contrib_ibl/samples/samples_htmlpage_2.json

@@ -240,6 +240,14 @@
         }, 
         {
                 "start": 2180, 
+                "end": 2182
+        },
+        {
+                "start": 2182,
+                "end": 2205
+        },
+        {
+                "start": 2205,
                 "end": 2206
         }, 
         {
@@ -298,6 +306,14 @@
         }, 
         {
                 "start": 2508, 
+                "end": 2509
+        },
+        {
+                "start": 2509,
+                "end": 2663
+        },
+        {
+                "start": 2663,
                 "end": 2664
         }, 
         {
@@ -356,7 +372,15 @@
                 "tag_type": 2
         }, 
         {
-                "start": 3086, 
+                "start": 3086,
+                "end": 3087
+        },
+        {
+                "start": 3087,
+                "end": 3108
+        },
+        {
+                "start": 3108,
                 "end": 3110
         }, 
         {
@@ -21765,4 +21789,4 @@
                 "start": 72321, 
                 "end": 72325
         }
-]
+]

scrapy/tests/test_contrib_ibl/test_htmlpage_data.py

@@ -165,7 +165,9 @@ PARSED3 = [
  {'attributes': {}, 'end': 55, 'start': 51, 'tag': u'p', 'tag_type': 2},
  {'end': 70, 'start': 55},
  {'attributes': {u'type': u'text/javascript'}, 'end': 101, 'start': 70, 'tag': u'script', 'tag_type': 1},
- {'end': 124, 'start': 101},
+ {'end': 104, 'start': 101},
+ {'end': 118, 'start': 104},
+ {'end': 124, 'start': 118},
  {'attributes': {}, 'end': 133, 'start': 124, 'tag': u'script', 'tag_type': 2},
  {'attributes': {}, 'end': 140, 'start': 133, 'tag': u'body', 'tag_type': 2},
  {'attributes': {}, 'end': 147, 'start': 140, 'tag': u'html', 'tag_type': 2}
@@ -204,7 +206,8 @@ PARSED5 = [
  {'end': 45, 'start': 42},
  {'attributes': {}, 'end': 54, 'start': 45, 'tag': u'script', 'tag_type': 2},
  {'attributes': {}, 'end': 61, 'start': 54, 'tag': u'body', 'tag_type': 2},
- {'end': 91, 'start': 61},
+ {'end': 76, 'start': 61},
+ {'end': 91, 'start': 76},
  {'attributes': {}, 'end': 98, 'start': 91, 'tag': u'html', 'tag_type': 2},
 ]
  
@@ -215,7 +218,9 @@ PARSED6 = [
  {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1},
  {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1},
  {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1},
- {'end': 40, 'start': 20},
+ {'end': 23, 'start': 20},
+ {'end': 37, 'start': 23},
+ {'end': 40, 'start': 37},
  {'attributes': {}, 'end': 49, 'start': 40, 'tag': u'script', 'tag_type': 2},
  {'end': 52, 'start': 49},
  {'attributes': {}, 'end': 60, 'start': 52, 'tag': u'script', 'tag_type': 1},
@@ -225,3 +230,19 @@ PARSED6 = [
  {'attributes': {}, 'end': 81, 'start': 74, 'tag': u'body', 'tag_type': 2},
  {'attributes': {}, 'end': 88, 'start': 81, 'tag': u'html', 'tag_type': 2},
 ]
+
+# Test source without ending body nor html
+PAGE7 = u"""<html><body><p>veris in temporibus sub aprilis idibus</p><script>script code</script><!--comment-->"""
+
+PARSED7 = [
+    {'attributes' : {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1},
+    {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1},
+    {'attributes': {}, 'end': 15, 'start': 12, 'tag': u'p', 'tag_type': 1},
+    {'end': 53, 'start': 15},
+    {'attributes': {}, 'end': 57, 'start': 53, 'tag': u'p', 'tag_type': 2},
+    {'attributes' : {}, 'end': 65, 'start': 57, 'tag': u'script', 'tag_type': 1},
+    {'end': 76, 'start': 65},
+    {'attributes' : {}, 'end': 85, 'start': 76, 'tag': u'script', 'tag_type': 2},
+    {'end': 99, 'start': 85},
+]
+