RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Skipped IBL tests if nltk/numpy are not available.

This commit is contained in:
Rolando Espinoza La fuente 2010-05-28 16:53:17 -03:00
parent a71dc295af
commit e995c5c7ff
2 changed files with 46 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
scrapy/tests/test_contrib_ibl/test_extraction.py
View File

@ -4,14 +4,26 @@ tests for page parsing
Page parsing effectiveness is measured through the evaluation system. These
tests should focus on specific bits of functionality work correctly.
"""
from unittest import TestCase
from twisted.trial.unittest import TestCase, SkipTest
from scrapy.contrib.ibl.htmlpage import HtmlPage
from scrapy.contrib.ibl.extraction import InstanceBasedLearningExtractor
from scrapy.contrib.ibl.descriptor import (FieldDescriptor as A,
ItemDescriptor)
from scrapy.contrib.ibl.extractors import (contains_any_numbers,
image_url)
try:
import nltk
except ImportError:
nltk = None
try:
import numpy
except ImportError:
numpy = None
if nltk and numpy:
from scrapy.contrib.ibl.extraction import InstanceBasedLearningExtractor
# simple page with all features
ANNOTATED_PAGE1 = u"""
@ -778,7 +790,13 @@ TEST_DATA = [
class TestExtraction(TestCase):
def _run(self, name, templates, page, extractors, expected_output):
def setUp(self):
if not nltk:
raise SkipTest("nltk not available")
if not numpy:
raise SkipTest("numpy not available")
def _run_extraction(self, name, templates, page, extractors, expected_output):
self.trace = None
template_pages = [HtmlPage(None, {}, t) for t in templates]
extractor = InstanceBasedLearningExtractor(template_pages, extractors, True)
@ -810,7 +828,7 @@ class TestExtraction(TestCase):
def test_expected_outputs(self):
try:
for data in TEST_DATA:
self._run(*data)
self._run_extraction(*data)
except AssertionError:
if self.trace:
print "Trace:"

27
scrapy/tests/test_contrib_ibl/test_pageparsing.py
View File

@ -5,16 +5,29 @@ import os
from cStringIO import StringIO
from gzip import GzipFile
from unittest import TestCase
from twisted.trial.unittest import TestCase, SkipTest
from scrapy.utils.python import str_to_unicode
from scrapy.utils.py26 import json
from scrapy.contrib.ibl.htmlpage import HtmlPage
from scrapy.contrib.ibl.extraction.pageparsing import (InstanceLearningParser,
TemplatePageParser, ExtractionPageParser)
from scrapy.contrib.ibl.extraction.pageobjects import TokenDict, TokenType
from scrapy.tests.test_contrib_ibl import path
try:
import nltk
except ImportError:
nltk = None
try:
import numpy
except ImportError:
numpy = None
if nltk and numpy:
from scrapy.contrib.ibl.extraction.pageparsing import (
InstanceLearningParser, TemplatePageParser, ExtractionPageParser)
from scrapy.contrib.ibl.extraction.pageobjects import TokenDict, TokenType
SIMPLE_PAGE = u"""
<html> <p some-attr="foo">this is a test</p> </html>
"""
@ -160,6 +173,12 @@ def _tags(pp, predicate):
class TestPageParsing(TestCase):
def setUp(self):
if not nltk:
raise SkipTest("nltk not available")
if not numpy:
raise SkipTest("numpy not available")
def test_instance_parsing(self):
pp = _parse_page(InstanceLearningParser, SIMPLE_PAGE)
# all tags