diff --git a/scrapy/tests/__init__.py b/scrapy/tests/__init__.py index caca0592e..a102cf994 100644 --- a/scrapy/tests/__init__.py +++ b/scrapy/tests/__init__.py @@ -8,3 +8,12 @@ To run all Scrapy unittests type: Keep in mind that some tests may be skipped if you don't have some (optional) modules available like MySQLdb or simplejson. """ + +import os + +tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data') + +def get_testdata(*paths): + """Return test data""" + path = os.path.join(tests_datadir, *paths) + return open(path).read() diff --git a/scrapy/tests/run.py b/scrapy/tests/run.py index 337b0578c..bc58cc747 100644 --- a/scrapy/tests/run.py +++ b/scrapy/tests/run.py @@ -3,11 +3,13 @@ Module to run Scrapy tests - see scrapy.tests docstring """ if __name__ == '__main__': - import os + import os, sys from twisted.trial import runner, reporter + tests_to_run = sys.argv[1:] or ['scrapy'] + os.environ['SCRAPY_SETTINGS_DISABLED'] = '1' loader = runner.TestLoader() runner = runner.TrialRunner(reporter.TreeReporter) - suite = loader.loadByNames(['scrapy'], recurse=True) + suite = loader.loadByNames(tests_to_run, recurse=True) runner.run(suite) diff --git a/scrapy/tests/test_adaptors.py b/scrapy/tests/test_adaptors.py index 07bc2c3cc..2de1e5740 100644 --- a/scrapy/tests/test_adaptors.py +++ b/scrapy/tests/test_adaptors.py @@ -1,5 +1,4 @@ # -*- coding: utf8 -*- -import os import unittest import re @@ -7,7 +6,7 @@ from scrapy.item.adaptors import AdaptorPipe from scrapy.contrib_exp import adaptors from scrapy.http import HtmlResponse, Headers from scrapy.xpath.selector import HtmlXPathSelector, XmlXPathSelector -from scrapy.link import Link +from scrapy.tests import get_testdata class AdaptorPipeTestCase(unittest.TestCase): def test_pipe_init(self): @@ -37,12 +36,9 @@ class AdaptorPipeTestCase(unittest.TestCase): self.assertTrue(isinstance(pipe1 + sample_callable, AdaptorPipe)) class AdaptorsTestCase(unittest.TestCase): - def setUp(self): - self.samplesdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample_data', 'adaptors')) def get_selector(self, domain, url, sample_filename, headers=None, selector=HtmlXPathSelector): - sample_filename = os.path.join(self.samplesdir, sample_filename) - body = file(sample_filename).read() + body = get_testdata('adaptors', sample_filename) response = HtmlResponse(url=url, headers=Headers(headers), status=200, body=body) return selector(response) diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index c140f2811..738afed9c 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -1,21 +1,14 @@ -import os from unittest import TestCase, main from scrapy.http import Response, XmlResponse from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware +from scrapy.tests import get_testdata def setUp(): - datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'compressed') formats = ['tar', 'xml.bz2', 'xml.gz', 'zip'] - - uncompressed_fd = open(os.path.join(datadir, 'feed-sample1.xml'), 'r') - uncompressed_body = uncompressed_fd.read() - uncompressed_fd.close() - + uncompressed_body = get_testdata('compressed', 'feed-sample1.xml') test_responses = {} for format in formats: - fd = open(os.path.join(datadir, 'feed-sample1.' + format), 'r') - body = fd.read() - fd.close() + body = get_testdata('compressed', 'feed-sample1.' + format) test_responses[format] = Response('http://foo.com/bar', body=body) return uncompressed_body, test_responses diff --git a/scrapy/tests/test_downloadermiddleware_httpcompression.py b/scrapy/tests/test_downloadermiddleware_httpcompression.py index 3d1c9f9c0..41829795d 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcompression.py +++ b/scrapy/tests/test_downloadermiddleware_httpcompression.py @@ -6,9 +6,10 @@ from os.path import join, abspath, dirname from scrapy.spider import spiders from scrapy.http import Response, Request from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware +from scrapy.tests import tests_datadir -SAMPLEDIR = join(abspath(dirname(__file__)), 'sample_data/compressed') +SAMPLEDIR = join(tests_datadir, 'compressed') FORMAT = { 'gzip': ('html-gzip.bin', 'gzip'), diff --git a/scrapy/tests/test_engine.py b/scrapy/tests/test_engine.py index 54d2771a0..dc9374c09 100644 --- a/scrapy/tests/test_engine.py +++ b/scrapy/tests/test_engine.py @@ -10,6 +10,8 @@ import unittest from twisted.internet import reactor from twisted.web import server, resource, static, util +from scrapy.tests import tests_datadir + #class TestResource(resource.Resource): # isLeaf = True # @@ -17,7 +19,7 @@ from twisted.web import server, resource, static, util # return "hello world!" def start_test_site(): - root_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "test_site") + root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) # r.putChild("test", TestResource()) r.putChild("redirect", util.Redirect("/redirected")) diff --git a/scrapy/tests/test_link.py b/scrapy/tests/test_link.py index d28c59795..ef89909bf 100644 --- a/scrapy/tests/test_link.py +++ b/scrapy/tests/test_link.py @@ -1,4 +1,3 @@ -import os import re import unittest @@ -6,6 +5,7 @@ from scrapy.http import HtmlResponse from scrapy.link import LinkExtractor, Link from scrapy.link.extractors import RegexLinkExtractor from scrapy.contrib.link_extractors import HTMLImageLinkExtractor +from scrapy.tests import get_testdata class LinkExtractorTestCase(unittest.TestCase): def test_basic(self): @@ -36,11 +36,10 @@ class LinkExtractorTestCase(unittest.TestCase): [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) def test_extraction_encoding(self): - base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') - body = open(os.path.join(base_path, 'linkextractor_noenc.html'), 'r').read() + body = get_testdata('link_extractor', 'linkextractor_noenc.html') response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']}) response_noenc = HtmlResponse(url='http://example.com/noenc', body=body) - body = open(os.path.join(base_path, 'linkextractor_latin1.html'), 'r').read() + body = get_testdata('link_extractor', 'linkextractor_latin1.html') response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body) lx = LinkExtractor() @@ -66,8 +65,7 @@ class LinkExtractorTestCase(unittest.TestCase): class RegexLinkExtractorTestCase(unittest.TestCase): def setUp(self): - base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') - body = open(os.path.join(base_path, 'regex_linkextractor.html'), 'r').read() + body = get_testdata('link_extractor', 'regex_linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def test_urls_type(self): @@ -202,8 +200,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase): class HTMLImageLinkExtractorTestCase(unittest.TestCase): def setUp(self): - base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor') - body = open(os.path.join(base_path, 'image_linkextractor.html'), 'r').read() + body = get_testdata('link_extractor', 'image_linkextractor.html') self.response = HtmlResponse(url='http://example.com/index', body=body) def tearDown(self): diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index 3f5b02c06..a4d655c26 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -4,6 +4,7 @@ import libxml2 from scrapy.utils.iterators import csviter, xmliter from scrapy.http import XmlResponse, TextResponse +from scrapy.tests import get_testdata class UtilsIteratorsTestCase(unittest.TestCase): ### NOTE: Encoding issues have been found with BeautifulSoup for utf-16 files, utf-16 test removed ### @@ -94,8 +95,7 @@ class UtilsCsvTestCase(unittest.TestCase): sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv') def test_csviter_defaults(self): - body = open(self.sample_feed_path).read() - + body = get_testdata('feeds', 'feed-sample3.csv') response = TextResponse(url="http://example.com/", body=body) csv = csviter(response) @@ -112,8 +112,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assert_(all((isinstance(v, unicode) for v in result_row.values()))) def test_csviter_delimiter(self): - body = open(self.sample_feed_path).read().replace(',', '\t') - + body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t') response = TextResponse(url="http://example.com/", body=body) csv = csviter(response, delimiter='\t') @@ -124,7 +123,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'4', u'name': u'empty', u'value': u''}]) def test_csviter_headers(self): - sample = open(self.sample_feed_path).read().splitlines() + sample = get_testdata('feeds', 'feed-sample3.csv').splitlines() headers, body = sample[0].split(','), '\n'.join(sample[1:]) response = TextResponse(url="http://example.com/", body=body) @@ -137,7 +136,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'4', u'name': u'empty', u'value': u''}]) def test_csviter_falserow(self): - body = open(self.sample_feed_path).read() + body = get_testdata('feeds', 'feed-sample3.csv') body = '\n'.join((body, 'a,b', 'a,b,c,d')) response = TextResponse(url="http://example.com/", body=body) @@ -150,7 +149,7 @@ class UtilsCsvTestCase(unittest.TestCase): {u'id': u'4', u'name': u'empty', u'value': u''}]) def test_csviter_exception(self): - body = open(self.sample_feed_path).read() + body = get_testdata('feeds', 'feed-sample3.csv') response = TextResponse(url="http://example.com/", body=body) iter = csviter(response) @@ -162,8 +161,8 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertRaises(StopIteration, iter.next) def test_csviter_encoding(self): - body1 = open(self.sample_feed2_path).read() - body2 = open(self.sample_feed3_path).read() + body1 = get_testdata('feeds', 'feed-sample4.csv') + body2 = get_testdata('feeds', 'feed-sample5.csv') response = TextResponse(url="http://example.com/", body=body1, encoding='latin1') csv = csviter(response)