COMPANY - Item Title
+introduction
+
+
+
+
+
+
+This is such a nice item
Everybody likes it.
+
+
click here for other items
+ +""" + +EXTRACT_PAGE1 = u""" + +Scrapy - Nice Product
+introduction
+
+
+
+
+
+"""
+
+# single tag with multiple items extracted
+ANNOTATED_PAGE2 = u"""
+xx
+xxx
+
+"""
+EXTRACT_PAGE2 = u"""product 1 is great"""
+
+# matching must match the second attribute in order to find the first
+ANNOTATED_PAGE3 = u"""
+
+wonderful product
++
xx
+xx
+"""
+EXTRACT_PAGE3 = u"""
+description
+delivery
+this is not the description
+""" + +# test inferring repeated elements +ANNOTATED_PAGE4 = u""" +-
+
- feature1 +
- feature2 +
-
+
- feature1 ignore this +
- feature2 +
- feature3 +
description
+| colour 1 | +price 1 | +
| colour 2 | +price 2 | +
description
+| colour 1 | +price 1 | +
| colour 2 | +price 2 | +
| colour 3 | +price 3 | +
description
+name 1
+name 3
+name 2
+""" +EXTRACT_PAGE6 = u""" +description
+name 1
+name 3
+name 2
+""" + +# test repeating variants at the table column level +ANNOTATED_PAGE7 = u""" +| colour 1 | +colour 2 | +
| price 1 | +price 2 | +
| colour 1 | +colour 2 | +colour 3 | +
| price 1 | +price 2 | +price 3 | +
A product
+
+
+XXXX XXXX xxxxx
+ +
+10.00
+
+"""
+
+EXTRACT_PAGE8 = u"""
+
+13
+A product
+
+
+A very nice product for all intelligent people
+ +
+12.00
+
+"""
+
+ANNOTATED_PAGE9 = ANNOTATED_PAGE8
+
+EXTRACT_PAGE9 = u"""
+
+ID 15
+(VAT exc.)
+A product
+
+
+A very nice product for all intelligent people
+ +
+12.00
+
+"""
+
+ANNOTATED_PAGE10a = u"""
+
+ID 16
+(VAT exc.)| SKU | L345 | +
| Size | 10cmx20cm | +
| Price | £s;99.00 | +
| SKU | S220 | +
| Size | 20cmx20cm | +
| Price | £s;85.00 | +
| Offer | From $2500.00 | +
| Description | Electrorheological Cyborgs | +
| Series: | T2000 | +
| SKU | K80 | +
| Size | 50cm | +
| Price | &euros;85.00 | +
+
+SL342
+
+
+Nice product for ladies
+
+£s;85.00
+
+
+SL342
+
+Nice product for ladies
+
+£s;85.00
+
A product
+ + +""" + +EXTRACT_PAGE12a = u""" + +A product
+
+
+
+"""
+
+EXTRACT_PAGE12b = u"""
+
+A very nice product for all intelligent people
+ +
+12.00
ID 15
+(VAT exc.) +A product
+
+
+
+"""
+
+# Ex1: nested annotation with token sequence replica outside exterior annotation
+# and a possible sequence pattern can be extracted only with
+# correct handling of nested annotations
+ANNOTATED_PAGE13a = u"""
+
+
+A very nice product for all intelligent people
+ +
+12.00
+ID 15
+(VAT exc.) +-
+Features
+
- Feature A +
- Feature B +
+
+
A product
+$50.00 +This product is excelent. Buy it! + + + ++
See other products:
+Product b + + ++ +""" + +EXTRACT_PAGE13a = u""" + + +
+
A product
+$50.00 +This product is excelent. Buy it! ++ + + +
+
See other products:
+Product B + + + +""" + +# Ex2: annotation with token sequence replica inside a previous nested annotation +# and a possible sequence pattern can be extracted only with +# correct handling of nested annotations +ANNOTATED_PAGE13b = u""" + + ++
A product
+Previous price: $50.00 +This product is excelent. Buy it! + + + ++
Save 10%!!
+$45.00 + + + +""" + +EXTRACT_PAGE13b = u""" + + ++
A product
+$50.00 +This product is excelent. Buy it! + + + ++
+
Save 10%!!
+$45.00 + + ++ +""" + +ANNOTATED_PAGE14 = u""" + + + + +""" + +EXTRACT_PAGE14 = u""" + + +""" + +ANNOTATED_PAGE15 = u""" + +
Short
+
+892342
++
Description +90.00 +
+ +""" + +EXTRACT_PAGE15 = u""" + ++
Description +80.00 +
+ +""" + +ANNOTATED_PAGE16 = u""" + +
+Description
+
+
+"""
+
+EXTRACT_PAGE16 = u"""
+
++name
++80.00
+product name
+90.00
+ +""" + +ANNOTATED_PAGE17 = u""" + + ++This product is excelent. Buy it! +
+ +
+
+See other products:
+Product b +
+ + +""" + +EXTRACT_PAGE17 = u""" + + ++This product is excelent. Buy it! +
+ +
+
+See other products:
+Product B +
+ + +""" + +ANNOTATED_PAGE18 = u""" + +
+Item Id
+
+Description +
+
+"""
+
+EXTRACT_PAGE18 = u"""
+
++Description +
+Item Id
+
+Description +
+
+"""
+
+ANNOTATED_PAGE19 = u"""
+
++Description +
+
+
+
+"""
+
+EXTRACT_PAGE19a = u"""
+
+Product name
+60.00
+
+description
+
+
+
+
+"""
+
+EXTRACT_PAGE19b = u"""
+
+Product name
+60.00
+
+description
+
+
+
+
+"""
+
+SAMPLE_DESCRIPTOR1 = ItemDescriptor('test', 'product test', [
+ A('name', "Product name", required=True),
+ A('price', "Product price, including any discounts and tax or vat",
+ contains_any_numbers, True),
+ A('image_urls', "URLs for one or more images", image_url, True),
+ A('description', "The full description of the product", allow_markup=True),
+ ]
+)
+
+# A list of (test name, [templates], page, extractors, expected_result)
+TEST_DATA = [
+ # extract from a similar page
+ ('similar page extraction', [ANNOTATED_PAGE1], EXTRACT_PAGE1, None,
+ {u'title': [u'Nice Product'], u'description': [u'wonderful product'],
+ u'image_url': [u'nice_product.jpg']}
+ ),
+ # strip the first 5 characters from the title
+ ('extractor test', [ANNOTATED_PAGE1], EXTRACT_PAGE1,
+ ItemDescriptor('test', 'product test',
+ [A('title', "something about a title", lambda x: x[5:])]),
+ {u'title': [u'Product'], u'description': [u'wonderful product'],
+ u'image_url': [u'nice_product.jpg']}
+ ),
+ # compilicated tag (multiple attributes and annotation)
+ ('multiple attributes and annotation', [ANNOTATED_PAGE2], EXTRACT_PAGE2, None,
+ {'name': [u'product 1'], 'image_url': [u'http://example.com/product1.jpg'],
+ 'description': [u'product 1 is great']}
+ ),
+ # can only work out correct placement by matching the second attribute first
+ ('ambiguous description', [ANNOTATED_PAGE3], EXTRACT_PAGE3, None,
+ {'description': [u'description'], 'delivery': [u'delivery']}
+ ),
+ # infer a repeated structure
+ ('repeated elements', [ANNOTATED_PAGE4], EXTRACT_PAGE4, None,
+ {'features': [u'feature1', u'feature2', u'feature3']}
+ ),
+ # identical variants with a repeated structure
+ ('repeated identical variants', [ANNOTATED_PAGE5], EXTRACT_PAGE5, None,
+ {
+ 'description': [u'description'],
+ 'variants': [
+ {u'colour': [u'colour 1'], u'price': [u'price 1']},
+ {u'colour': [u'colour 2'], u'price': [u'price 2']},
+ {u'colour': [u'colour 3'], u'price': [u'price 3']}
+ ]
+ }
+ ),
+ # variants with an irregular structure
+ ('irregular variants', [ANNOTATED_PAGE6], EXTRACT_PAGE6, None,
+ {
+ 'description': [u'description'],
+ 'variants': [
+ {u'name': [u'name 1']},
+ {u'name': [u'name 3']},
+ {u'name': [u'name 2']}
+ ]
+ }
+ ),
+
+ # discovering repeated variants in table columns
+# ('variants in table columns', [ANNOTATED_PAGE7], EXTRACT_PAGE7, None,
+# {'variants': [
+# {u'colour': [u'colour 1'], u'price': [u'price 1']},
+# {u'colour': [u'colour 2'], u'price': [u'price 2']},
+# {u'colour': [u'colour 3'], u'price': [u'price 3']}
+# ]}
+# ),
+
+
+ # ignored regions
+ (
+ 'ignored_regions', [ANNOTATED_PAGE8], EXTRACT_PAGE8, None,
+ {
+ 'description': [u'\n A very nice product for all intelligent people \n\n'],
+ 'price': [u'\n12.00\n(VAT exc.)'],
+ }
+ ),
+ # shifted ignored regions (detected by region similarity)
+ (
+ 'shifted_ignored_regions', [ANNOTATED_PAGE9], EXTRACT_PAGE9, None,
+ {
+ 'description': [u'\n A very nice product for all intelligent people \n\n'],
+ 'price': [u'\n12.00\n(VAT exc.)'],
+ }
+ ),
+ # detection of common prefixes across templates, all templates marked
+ (# wrong extraction
+ 'without_match_common_prefix', [ANNOTATED_PAGE10a], EXTRACT_PAGE10a, None,
+ {
+ 'price': [u'\n Series: T2000 \n'],
+ 'dimensions': [u'\n Description Electrorheological Cyborgs \n'],
+ 'site_id': [u'\n Offer From $2500.00 \n'],
+ }
+ ),
+ (# right extraction
+ 'with_match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10a, None,
+ {}
+ ),
+ (# another example
+ 'match_common_prefix', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b, None,
+ {
+ 'dimensions': [u'50cm'],
+ 'site_id': [u'K80'],
+ }
+ ),
+ (# common_prefix with allow_markup attribute
+ 'common_prefix_allow_markup', [ANNOTATED_PAGE10a, ANNOTATED_PAGE10b], EXTRACT_PAGE10b,
+ ItemDescriptor('test', 'product test',
+ [A('dimensions', "something about dimensions", allow_markup=True)]),
+ {
+ 'dimensions': [u'50cm'],
+ 'site_id': [u'K80'],
+ }
+ ),
+ (# special case with partial annotations
+ 'special_partial_annotation', [ANNOTATED_PAGE11], EXTRACT_PAGE11, None,
+ {
+ 'name': [u'SL342'],
+ 'description': [u'\nSL342\n \nNice product for ladies\n \n£s;85.00\n'],
+ 'price': [u'£s;85.00'],
+ 'price_before_discount': [u'£s;100.00'],
+ }
+ ),
+ (# with ignore-beneath feature
+ 'ignore-beneath', [ANNOTATED_PAGE12], EXTRACT_PAGE12a, None,
+ {
+ 'description': [u'\n A very nice product for all intelligent people \n'],
+ }
+ ),
+ (# ignore-beneath with extra tags
+ 'ignore-beneath with extra tags', [ANNOTATED_PAGE12], EXTRACT_PAGE12b, None,
+ {
+ 'description': [u'\n A very nice product for all intelligent people \n'],
+ }
+ ),
+ ('nested annotation with replica outside', [ANNOTATED_PAGE13a], EXTRACT_PAGE13a, None,
+ {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n \n'],
+ 'price': ["$50.00"],
+ 'name': [u'A product']}
+ ),
+ ('outside annotation with nested replica', [ANNOTATED_PAGE13b], EXTRACT_PAGE13b, None,
+ {'description': [u'\n A product \n $50.00 \nThis product is excelent. Buy it!\n'],
+ 'price': ["$45.00"],
+ 'name': [u'A product']}
+ ),
+ ('consistency check', [ANNOTATED_PAGE14], EXTRACT_PAGE14, None,
+ {},
+ ),
+ ('consecutive nesting', [ANNOTATED_PAGE15], EXTRACT_PAGE15, None,
+ {'description': [u'Description\n\n'],
+ 'price': [u'80.00']},
+ ),
+ ('nested inside not found', [ANNOTATED_PAGE16], EXTRACT_PAGE16, None,
+ {'price': [u'90.00'],
+ 'name': [u'product name']},
+ ),
+ ('ignored region helps to find attributes', [ANNOTATED_PAGE17], EXTRACT_PAGE17, None,
+ {'description': [u'\nThis product is excelent. Buy it!\n']},
+ ),
+ ('ignored region in partial annotation', [ANNOTATED_PAGE18], EXTRACT_PAGE18, None,
+ {u'site_id': [u'Item Id'],
+ u'description': [u'\nDescription\n']},
+ ),
+ ('extra required attribute product', [ANNOTATED_PAGE19], EXTRACT_PAGE19a,
+ SAMPLE_DESCRIPTOR1,
+ {u'price': [u'60.00'],
+ u'description': [u'description'],
+ u'image_urls': [['http://example.com/image.jpg']],
+ u'name': [u'Product name']},
+ ),
+ ('extra required attribute no product', [ANNOTATED_PAGE19], EXTRACT_PAGE19b,
+ SAMPLE_DESCRIPTOR1,
+ None,
+ ),
+]
+
+class TestExtraction(TestCase):
+
+ def _run(self, name, templates, page, extractors, expected_output):
+ self.trace = None
+ template_pages = [HtmlPage(None, {}, t) for t in templates]
+ extractor = InstanceBasedLearningExtractor(template_pages, extractors, True)
+ actual_output, _ = extractor.extract(HtmlPage(None, {}, page))
+ if not actual_output:
+ if expected_output is None:
+ return
+ assert False, "failed to extract data for test '%s'" % name
+ actual_output = actual_output[0]
+ self.trace = ["Extractor:\n%s" % extractor] + actual_output.pop('trace', [])
+ expected_names = set(expected_output.keys())
+ actual_names = set(actual_output.keys())
+
+ missing_in_output = filter(None, expected_names - actual_names)
+ error = "attributes '%s' were expected but were not present in test '%s'" % \
+ ("', '".join(missing_in_output), name)
+ assert len(missing_in_output) == 0, error
+
+ unexpected = actual_names - expected_names
+ error = "unexpected attributes %s in test '%s'" % \
+ (', '.join(unexpected), name)
+ assert len(unexpected) == 0, error
+
+ for k, v in expected_output.items():
+ extracted = actual_output[k]
+ assert v == extracted, "in test '%s' for attribute '%s', " \
+ "expected value '%s' but got '%s'" % (name, k, v, extracted)
+
+ def test_expected_outputs(self):
+ try:
+ for data in TEST_DATA:
+ self._run(*data)
+ except AssertionError:
+ if self.trace:
+ print "Trace:"
+ for line in self.trace:
+ print "\n---\n%s" % line
+ raise
diff --git a/scrapy/tests/test_contrib_ibl/test_htmlpage.py b/scrapy/tests/test_contrib_ibl/test_htmlpage.py
new file mode 100644
index 000000000..f502c6364
--- /dev/null
+++ b/scrapy/tests/test_contrib_ibl/test_htmlpage.py
@@ -0,0 +1,134 @@
+"""
+htmlpage.py tests
+"""
+import os
+from gzip import GzipFile
+from unittest import TestCase
+
+from scrapy.utils.py26 import json
+from scrapy.tests.test_contrib_ibl import path
+from scrapy.contrib.ibl.htmlpage import parse_html, HtmlTag, HtmlDataFragment
+from scrapy.tests.test_contrib_ibl.test_htmlpage_data import *
+
+SAMPLES_FILE = "samples_htmlpage.json.gz"
+
+def _encode_element(el):
+ """
+ jsonize parse element
+ """
+ if isinstance(el, HtmlTag):
+ return {"tag": el.tag, "attributes": el.attributes,
+ "start": el.start, "end": el.end, "tag_type": el.tag_type}
+ if isinstance(el, HtmlDataFragment):
+ return {"start": el.start, "end": el.end}
+ raise TypeError
+
+def _decode_element(dct):
+ """
+ dejsonize parse element
+ """
+ if "tag" in dct:
+ return HtmlTag(dct["tag_type"], dct["tag"], \
+ dct["attributes"], dct["start"], dct["end"])
+ if "start" in dct:
+ return HtmlDataFragment(dct["start"], dct["end"])
+ return dct
+
+def add_sample(source):
+ """
+ Method for adding samples to test samples file
+ (use from console)
+ """
+ samples = []
+ if os.path.exists(SAMPLES_FILE):
+ for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines():
+ samples.append(json.loads(line))
+
+ new_sample = {"source": source}
+ new_sample["parsed"] = list(parse_html(source))
+ samples.append(new_sample)
+ samples_file = GzipFile(os.path.join(path, SAMPLES_FILE), "wb")
+ for sample in samples:
+ samples_file.write(json.dumps(sample, default=_encode_element) + "\n")
+ samples_file.close()
+
+class TestParseHtml(TestCase):
+ """Test for parse_html"""
+ def _test_sample(self, sample):
+ source = sample["source"]
+ expected_parsed = sample["parsed"]
+ parsed = parse_html(source)
+ count_element = 0
+ count_expected = 0
+ for element in parsed:
+ if type(element) == HtmlTag:
+ count_element += 1
+ expected = expected_parsed.pop(0)
+ if type(expected) == HtmlTag:
+ count_expected += 1
+ element_text = source[element.start:element.end]
+ expected_text = source[expected.start:expected.end]
+ if element.start != expected.start or element.end != expected.end:
+ assert False, "[%s,%s] %s != [%s,%s] %s" % (element.start, \
+ element.end, element_text, expected.start, \
+ expected.end, expected_text)
+ if type(element) != type(expected):
+ assert False, "(%s) %s != (%s) %s for text\n%s" % (count_element, \
+ repr(type(element)), count_expected, repr(type(expected)), element_text)
+ if type(element) == HtmlTag:
+ self.assertEqual(element.tag, expected.tag)
+ self.assertEqual(element.attributes, expected.attributes)
+
+ def test_parse(self):
+ """simple parse_html test"""
+ parsed = [_decode_element(d) for d in PARSED]
+ sample = {"source": PAGE, "parsed": parsed}
+ self._test_sample(sample)
+
+ def test_site_samples(self):
+ """test parse_html from real cases"""
+ samples = []
+ for line in GzipFile(os.path.join(path, SAMPLES_FILE), "r").readlines():
+ samples.append(json.loads(line, object_hook=_decode_element))
+ for sample in samples:
+ self._test_sample(sample)
+
+ def test_bad(self):
+ """test parsing of bad html layout"""
+ parsed = [_decode_element(d) for d in PARSED2]
+ sample = {"source": PAGE2, "parsed": parsed}
+ self._test_sample(sample)
+
+ def test_comments(self):
+ """test parsing of tags inside comments"""
+ parsed = [_decode_element(d) for d in PARSED3]
+ sample = {"source": PAGE3, "parsed": parsed}
+ self._test_sample(sample)
+
+ def test_script_text(self):
+ """test parsing of tags inside scripts"""
+ parsed = [_decode_element(d) for d in PARSED4]
+ sample = {"source": PAGE4, "parsed": parsed}
+ self._test_sample(sample)
+
+ def test_sucessive(self):
+ """test parsing of sucesive cleaned elements"""
+ parsed = [_decode_element(d) for d in PARSED5]
+ sample = {"source": PAGE5, "parsed": parsed}
+ self._test_sample(sample)
+
+ def test_sucessive2(self):
+ """test parsing of sucesive cleaned elements (variant 2)"""
+ parsed = [_decode_element(d) for d in PARSED6]
+ sample = {"source": PAGE6, "parsed": parsed}
+ self._test_sample(sample)
+
+ def test_special_cases(self):
+ """some special cases tests"""
+ parsed = list(parse_html(""))
+ self.assertEqual(parsed[0].attributes, {'content': 'no-cache', 'http-equiv': 'Pragma'})
+ parsed = list(parse_html(""))
+ self.assertEqual(parsed[0].attributes, {'xmlns': 'http://www.w3.org/1999/xhtml', 'xml:lang': 'en', 'lang': 'en'})
+ parsed = list(parse_html("Range
+from 20.00
+
+
+
+
"))
+ self.assertEqual(parsed[0].attributes, {'src': 'http://images.play.com/banners/SAM550a.jpg', \
+ 'align': 'left', 'hspace': '5', '/': None})
diff --git a/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py b/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py
new file mode 100644
index 000000000..2eac24f26
--- /dev/null
+++ b/scrapy/tests/test_contrib_ibl/test_htmlpage_data.py
@@ -0,0 +1,227 @@
+PAGE = u"""
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+LABELLED_PAGE3 = u"""
+
+
COMPANY - Item Title
+introduction
+
+
+
+
+
+
+This is such a nice item
Everybody likes it.
+
+
Power: 50W
+ +-
+
- Small +
- Big +
click here for other items
+Louis Chair
+| Height | +32.00 | +
+""" + +PARSED = [ +{'start': 0, 'end': 1}, +{'attributes': {'type': 'text/css', 'id': 'scrapy-style'}, 'tag': 'style', 'end': 42, 'start': 1, 'tag_type': 1}, +{'start': 42, 'end': 129}, +{'attributes': {}, 'tag': 'style', 'end': 137, 'start': 129, 'tag_type': 2}, +{'start': 137, 'end': 138}, +{'attributes': {}, 'tag': 'body', 'end': 144, 'start': 138, 'tag_type': 1}, +{'start': 144, 'end': 145}, +{'attributes': {'class': 'scrapy-selected', 'id': 'header'}, 'tag': 'div', 'end': 186, 'start': 145, 'tag_type': 1}, +{'start': 186, 'end': 187}, +{'attributes': {'src': 'company_logo.jpg', 'style': 'margin-left: 68px; padding-top:5px;', 'width': '530', 'alt': 'Logo', 'height': '105'}, 'tag': 'img', 'end': 295, 'start': 187, 'tag_type': 1}, +{'start': 295, 'end': 296}, +{'attributes': {'id': 'vertrule'}, 'tag': 'div', 'end': 315, 'start': 296, 'tag_type': 1}, +{'start': 315, 'end': 316}, +{'attributes': {}, 'tag': 'h1', 'end': 320, 'start': 316, 'tag_type': 1}, +{'start': 320, 'end': 330}, +{'attributes': {'data-scrapy-annotate': '{"variant": "0", "generated": true, "annotations": {"content": "title"}}'}, 'tag': 'ins', 'end': 491, 'start': 330, 'tag_type': 1}, +{'start': 491, 'end': 501}, +{'attributes': {}, 'tag': 'ins', 'end': 507, 'start': 501, 'tag_type': 2}, +{'attributes': {}, 'tag': 'h1', 'end': 512, 'start': 507, 'tag_type': 2}, +{'start': 512, 'end': 513}, +{'attributes': {}, 'tag': 'p', 'end': 516, 'start': 513, 'tag_type': 1}, +{'start': 516, 'end': 528}, +{'attributes': {}, 'tag': 'p', 'end': 532, 'start': 528, 'tag_type': 2}, +{'start': 532, 'end': 533}, +{'attributes': {}, 'tag': 'div', 'end': 538, 'start': 533, 'tag_type': 1}, +{'start': 538, 'end': 539}, +{'attributes': {'classid': None, 'src': '/upload/img.jpg', 'data-scrapy-annotate': '{"variant": "0", "annotations": {"image_url": "src"}}'}, 'tag': 'img', 'end': 709, 'start': 539, 'tag_type': 1}, +{'start': 709, 'end': 710}, +{'attributes': {'classid': None, 'data-scrapy-annotate': '{"variant": "0", "annotations": {"content": "description"}}'}, 'tag': 'p', 'end': 858, 'start': 710, 'tag_type': 1}, +{'start': 858, 'end': 883}, +{'attributes': {}, 'tag': 'br', 'end': 888, 'start': 883, 'tag_type': 3}, +{'start': 888, 'end': 909}, +{'attributes': {}, 'tag': 'p', 'end': 913, 'start': 909, 'tag_type': 2}, +{'start': 913, 'end': 914}, +{'attributes': {}, 'tag': 'br', 'end': 918, 'start': 914, 'tag_type': 1}, +{'attributes': {}, 'tag': 'br', 'end': 923, 'start': 918, 'tag_type': 2}, +{'start': 923, 'end': 924}, +{'attributes': {}, 'tag': 'div', 'end': 930, 'start': 924, 'tag_type': 2}, +{'start': 930, 'end': 931}, +{'attributes': {'data-scrapy-annotate': '{"variant": "0", "annotations": {"content": "features"}}', 'class': None}, 'tag': 'p', 'end': 1074, 'start': 931, 'tag_type': 1}, +{'start': 1074, 'end': 1084}, +{'attributes': {}, 'tag': 'p', 'end': 1088, 'start': 1084, 'tag_type': 2}, +{'start': 1088, 'end': 1109}, +{'attributes': {'data-scrapy-replacement': 'select', 'class': 'product'}, 'tag': 'ul', 'end': 1162, 'start': 1109, 'tag_type': 1}, +{'start': 1162, 'end': 1163}, +{'attributes': {'data-scrapy-replacement': 'option'}, 'tag': 'li', 'end': 1200, 'start': 1163, 'tag_type': 1}, +{'start': 1200, 'end': 1205}, +{'attributes': {}, 'tag': 'li', 'end': 1210, 'start': 1205, 'tag_type': 2}, +{'start': 1210, 'end': 1211}, +{'attributes': {'data-scrapy-replacement': 'option'}, 'tag': 'li', 'end': 1248, 'start': 1211, 'tag_type': 1}, +{'start': 1248, 'end': 1251}, +{'attributes': {}, 'tag': 'li', 'end': 1256, 'start': 1251, 'tag_type': 2}, +{'start': 1256, 'end': 1257}, +{'attributes': {}, 'tag': 'ul', 'end': 1262, 'start': 1257, 'tag_type': 2}, +{'start': 1262, 'end': 1263}, +{'attributes': {}, 'tag': 'p', 'end': 1266, 'start': 1263, 'tag_type': 1}, +{'start': 1266, 'end': 1292}, +{'attributes': {}, 'tag': 'p', 'end': 1296, 'start': 1292, 'tag_type': 2}, +{'start': 1296, 'end': 1297}, +{'attributes': {}, 'tag': 'h3', 'end': 1301, 'start': 1297, 'tag_type': 1}, +{'start': 1301, 'end': 1312}, +{'attributes': {}, 'tag': 'h3', 'end': 1317, 'start': 1312, 'tag_type': 2}, +{'start': 1317, 'end': 1318}, +{'attributes': {'cellpadding': '0', 'width': '420', 'cellspacing': '0', 'class': 'rulet'}, 'tag': 'table', 'end': 1383, 'start': 1318, 'tag_type': 1}, +{'attributes': {}, 'tag': 'tbody', 'end': 1390, 'start': 1383, 'tag_type': 1}, +{'start': 1390, 'end': 1391}, +{'attributes': {}, 'tag': 'tr', 'end': 1395, 'start': 1391, 'tag_type': 1}, +{'attributes': {}, 'tag': 'td', 'end': 1399, 'start': 1395, 'tag_type': 1}, +{'start': 1399, 'end': 1405}, +{'attributes': {}, 'tag': 'td', 'end': 1410, 'start': 1405, 'tag_type': 2}, +{'start': 1410, 'end': 1411}, +{'attributes': {}, 'tag': 'td', 'end': 1415, 'start': 1411, 'tag_type': 1}, +{'attributes': {'data-scrapy-annotate': '{"variant": "0", "generated": true, "annotations": {"content": "price"}}'}, 'tag': 'ins', 'end': 1576, 'start': 1415, 'tag_type': 1}, +{'start': 1576, 'end': 1581}, +{'attributes': {}, 'tag': 'ins', 'end': 1587, 'start': 1581, 'tag_type': 2}, +{'attributes': {}, 'tag': 'td', 'end': 1592, 'start': 1587, 'tag_type': 2}, +{'start': 1592, 'end': 1593}, +{'attributes': {}, 'tag': 'tr', 'end': 1598, 'start': 1593, 'tag_type': 2}, +{'attributes': {}, 'tag': 'tbody', 'end': 1605, 'start': 1598, 'tag_type': 1}, +{'attributes': {}, 'tag': 'table', 'end': 1613, 'start': 1605, 'tag_type': 2}, +{'start': 1613, 'end': 1614}, +{'attributes': {'style': 'my style', 'onmouseover': 'xxx', 'class': None}, 'tag': 'p', 'end': 1659, 'start': 1614, 'tag_type': 1}, +{'start': 1659, 'end': 1660}, +] + +# for testing parsing of some invalid html code (but still managed by browsers) +PAGE2 = u""" + +
++Hello world! + +
+ + +""" + +PARSED2 = [ + {'end': 1, 'start': 0}, + {'attributes': {}, 'end': 7, 'start': 1, 'tag': u'html', 'tag_type': 1}, + {'end': 8, 'start': 7}, + {'attributes': {}, 'end': 14, 'start': 8, 'tag': u'body', 'tag_type': 1}, + {'end': 15, 'start': 14}, + {'attributes': {u'style': u'"margin:', u'0pt"': None, u'class': u'"MsoNormal"', u'0cm': None}, 'end': 80, 'start': 15, 'tag': u'p', 'tag_type': 2}, + {'attributes': {u'lang': u'"EN-GB"'}, 'end': 107, 'start': 80, 'tag': u'span', 'tag_type': 1}, + {'end': 121, 'start': 107}, + {'attributes': {}, 'end': 128, 'start': 121, 'tag': u'span', 'tag_type': 2}, + {'end': 129, 'start': 128}, + {'attributes': {}, 'end': 133, 'start': 129, 'tag': u'p', 'tag_type': 2}, + {'end': 134, 'start': 133}, + {'attributes': {}, 'end': 141, 'start': 134, 'tag': u'body', 'tag_type': 2}, + {'end': 142, 'start': 141}, + {'attributes': {}, 'end': 149, 'start': 142, 'tag': u'html', 'tag_type': 2}, + {'end': 150, 'start': 149}, +] + +# for testing tags inside comments +PAGE3 = u"""Helloooo!!
Did i say hello??
""" + +PARSED3 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1}, + {'end': 26, 'start': 16}, + {'attributes': {}, 'end': 31, 'start': 26, 'tag': u'h1', 'tag_type': 2}, + {'attributes': {}, 'end': 34, 'start': 31, 'tag': u'p', 'tag_type': 1}, + {'end': 51, 'start': 34}, + {'attributes': {}, 'end': 55, 'start': 51, 'tag': u'p', 'tag_type': 2}, + {'end': 70, 'start': 55}, + {'attributes': {u'type': u'text/javascript'}, 'end': 101, 'start': 70, 'tag': u'script', 'tag_type': 1}, + {'end': 124, 'start': 101}, + {'attributes': {}, 'end': 133, 'start': 124, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 140, 'start': 133, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 147, 'start': 140, 'tag': u'html', 'tag_type': 2} +] + +# for testing tags inside scripts +PAGE4 = u"""Konnichiwa!!
hellohello again""" + +PARSED4 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 16, 'start': 12, 'tag': u'h1', 'tag_type': 1}, + {'end': 28,'start': 16}, + {'attributes': {}, 'end': 33, 'start': 28, 'tag': u'h1', 'tag_type': 2}, + {'end': 38, 'start': 33}, + {'attributes': {u'type': u'text/javascript'}, 'end': 69, 'start': 38, 'tag': u'script', 'tag_type': 1}, + {'end': 130, 'start': 69}, + {'attributes': {}, 'end': 139, 'start': 130, 'tag': u'script', 'tag_type': 2}, + {'end': 150, 'start': 139}, + {'attributes': {}, 'end': 157, 'start': 150, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 164, 'start': 157, 'tag': u'html', 'tag_type': 2}, +] + +# Test sucessive cleaning elements +PAGE5 = u"""""" + +PARSED5 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1}, + {'end': 25, 'start': 20}, + {'attributes': {}, 'end': 34, 'start': 25, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 42, 'start': 34, 'tag': u'script', 'tag_type': 1}, + {'end': 45, 'start': 42}, + {'attributes': {}, 'end': 54, 'start': 45, 'tag': u'script', 'tag_type': 2}, + {'attributes': {}, 'end': 61, 'start': 54, 'tag': u'body', 'tag_type': 2}, + {'end': 91, 'start': 61}, + {'attributes': {}, 'end': 98, 'start': 91, 'tag': u'html', 'tag_type': 2}, +] + +# Test sucessive cleaning elements variant 2 +PAGE6 = u"""all\n\n""" + +PARSED6 = [ + {'attributes': {}, 'end': 6, 'start': 0, 'tag': u'html', 'tag_type': 1}, + {'attributes': {}, 'end': 12, 'start': 6, 'tag': u'body', 'tag_type': 1}, + {'attributes': {}, 'end': 20, 'start': 12, 'tag': u'script', 'tag_type': 1}, + {'end': 40, 'start': 20}, + {'attributes': {}, 'end': 49, 'start': 40, 'tag': u'script', 'tag_type': 2}, + {'end': 52, 'start': 49}, + {'attributes': {}, 'end': 60, 'start': 52, 'tag': u'script', 'tag_type': 1}, + {'end': 63, 'start': 60}, + {'attributes': {}, 'end': 72, 'start': 63, 'tag': u'script', 'tag_type': 2}, + {'end': 74, 'start': 72}, + {'attributes': {}, 'end': 81, 'start': 74, 'tag': u'body', 'tag_type': 2}, + {'attributes': {}, 'end': 88, 'start': 81, 'tag': u'html', 'tag_type': 2}, +] diff --git a/scrapy/tests/test_contrib_ibl/test_pageparsing.py b/scrapy/tests/test_contrib_ibl/test_pageparsing.py new file mode 100644 index 000000000..545e9d46b --- /dev/null +++ b/scrapy/tests/test_contrib_ibl/test_pageparsing.py @@ -0,0 +1,283 @@ +""" +Unit tests for pageparsing +""" +import os +from cStringIO import StringIO +from gzip import GzipFile + +from unittest import TestCase +from scrapy.utils.python import str_to_unicode +from scrapy.utils.py26 import json + +from scrapy.contrib.ibl.htmlpage import HtmlPage +from scrapy.contrib.ibl.extraction.pageparsing import (InstanceLearningParser, + TemplatePageParser, ExtractionPageParser) +from scrapy.contrib.ibl.extraction.pageobjects import TokenDict, TokenType +from scrapy.tests.test_contrib_ibl import path + +SIMPLE_PAGE = u""" +this is a test
+""" + +LABELLED_PAGE1 = u""" + +Some Product
+some stuff
+
+This is such a nice item
+Everybody likes it.
+
+
+Old fashioned product +
+For exigent individuals +
click here for other items
+ +""" + +BROKEN_PAGE = u""" +html parser cannot parse this
+""" + +LABELLED_PAGE2 = u""" + +A product
+
+
+A very nice product for all intelligent people
+ +
+\xa310.00
+ 13
+
+
+
+
+
+"""
+
+LABELLED_PAGE3 = u"""
+
+A product
+
+
+A very nice product for all intelligent people
+ +
+\xa310.00
+
+
+"""
+
+LABELLED_PAGE4 = u"""
+
+ 13
| Description 1 | +Description 2 | +Description 3 | +Description 4 | +
A product
+
+
+A very nice product for all intelligent people
+ +
+\xa310.00
+
+
+"""
+
+LABELLED_PAGE5 = u"""
+
+ 13
| Description 1 | +Description 2 | +Description 3 | ++Price \xa310.00 | +
-
+
- Option A +
- Option I +
- Option B +
+65.00pounds
+Description: +Text B
+Text C + +""" + +LABELLED_PAGE7 = u""" + +
+Item Id
+Description
+
+
+"""
+
+LABELLED_PAGE8 = u"""
+
+
+Item Id
+Description
+
+
+"""
+
+def _parse_page(parser_class, pagetext):
+ htmlpage = HtmlPage(None, {}, pagetext)
+ parser = parser_class(TokenDict())
+ parser.feed(htmlpage)
+ return parser
+
+def _tags(pp, predicate):
+ return [pp.token_dict.token_string(s) for s in pp.token_list \
+ if predicate(s)]
+
+class TestPageParsing(TestCase):
+
+ def test_instance_parsing(self):
+ pp = _parse_page(InstanceLearningParser, SIMPLE_PAGE)
+ # all tags
+ self.assertEqual(_tags(pp, bool), ['', '', '
', '']) + + # open/closing tag handling + openp = lambda x: pp.token_dict.token_type(x) == TokenType.OPEN_TAG + self.assertEqual(_tags(pp, openp), ['', '']) + closep = lambda x: pp.token_dict.token_type(x) == TokenType.CLOSE_TAG + self.assertEqual(_tags(pp, closep), ['
', '']) + + def _validate_annotation(self, parser, lable_region, name, start_tag, end_tag): + assert lable_region.surrounds_attribute == name + start_token = parser.token_list[lable_region.start_index] + assert parser.token_dict.token_string(start_token) == start_tag + end_token = parser.token_list[lable_region.end_index] + assert parser.token_dict.token_string(end_token) == end_tag + + def test_template_parsing(self): + lp = _parse_page(TemplatePageParser, LABELLED_PAGE1) + self.assertEqual(len(lp.annotations), 5) + self._validate_annotation(lp, lp.annotations[0], + 'name', '', '
') + self.assertEqual(lp.annotations[0].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[1], + 'description', '', '
') + self.assertEqual(lp.annotations[1].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[2], + 'price', '', '') + self.assertEqual(lp.annotations[2].match_common_prefix, True) + self._validate_annotation(lp, lp.annotations[3], + 'short_description', '
', '
') + self.assertEqual(lp.annotations[3].match_common_prefix, False) + self._validate_annotation(lp, lp.annotations[4], + 'short_description', '
', '
') + self.assertEqual(lp.annotations[4].match_common_prefix, False) + + # all tags were closed + self.assertEqual(len(lp.labelled_tag_stacks), 0) + + def test_extraction_page_parsing(self): + epp = _parse_page(ExtractionPageParser, SIMPLE_PAGE) + ep = epp.to_extraction_page() + assert len(ep.page_tokens) == 4 + assert ep.token_html(0) == '' + assert ep.token_html(1) == '
' + + assert ep.html_between_tokens(1, 2) == 'this is a test' + assert ep.html_between_tokens(1, 3) == 'this is a test
' + + def test_invalid_html(self): + p = _parse_page(InstanceLearningParser, BROKEN_PAGE) + assert p + + def test_ignore_region(self): + """Test ignored regions""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE2) + self.assertEqual(p.ignored_regions, [(7,12),(15,17),(19,26),(21,22),(27,28),(28,29),(29,None),(30,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_ignore_regions2(self): + """Test ignore-beneath regions""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE3) + self.assertEqual(p.ignored_regions, [(7,12),(15,17),(22,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_ignore_regions3(self): + """Test ignore-beneath with annotation inside region""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE4) + self.assertEqual(p.ignored_regions, [(15,17),(22,None)]) + self.assertEqual(len(p.ignored_tag_stacks), 0) + + def test_replacement(self): + """Test parsing of replacement tags""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE5) + self.assertEqual(_tags(p, bool), ['', '', '', '', '']) + + def test_partial(self): + """Test partial annotation parsing""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE6) + text = p.annotations[0].annotation_text + self.assertEqual(text.start_text, '') + self.assertEqual(text.follow_text, 'pounds') + text = p.annotations[1].annotation_text + self.assertEqual(text.start_text, "Description: ") + self.assertEqual(text.follow_text, '') + + def test_ignored_partial(self): + """Test ignored region declared on partial annotation""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE7) + self.assertEqual(p.ignored_regions, [(2, 3)]) + + def test_extra_required(self): + """Test parsing of extra required attributes""" + p = _parse_page(TemplatePageParser, LABELLED_PAGE8) + self.assertEqual(p.extra_required_attrs, ["description"]) + + def test_site_pages(self): + """ + Tests from real pages. More reliable and easy to build for more complicated structures + """ + samples_file = open(os.path.join(path, "samples_pageparsing.json.gz"), "r") + samples = [] + for line in GzipFile(fileobj=StringIO(samples_file.read())).readlines(): + samples.append(json.loads(line)) + for sample in samples: + source = sample["annotated"] + annotations = sample["annotations"] + template = HtmlPage(body=str_to_unicode(source)) + parser = TemplatePageParser(TokenDict()) + parser.feed(template) + for annotation in parser.annotations: + test_annotation = annotations.pop(0) + for s in annotation.__slots__: + if s == "tag_attributes": + for pair in getattr(annotation, s): + self.assertEqual(list(pair), test_annotation[s].pop(0)) + else: + self.assertEqual(getattr(annotation, s), test_annotation[s]) + self.assertEqual(annotations, [])