sorted out some tests sample data paths and fixed bug with test in windows

scrapy/tests/__init__.py

@@ -8,3 +8,12 @@ To run all Scrapy unittests type:
 Keep in mind that some tests may be skipped if you don't have some (optional)
 modules available like MySQLdb or simplejson.
 """
+
+import os
+
+tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data')
+
+def get_testdata(*paths):
+    """Return test data"""
+    path = os.path.join(tests_datadir, *paths)
+    return open(path).read()

scrapy/tests/run.py

@@ -3,11 +3,13 @@ Module to run Scrapy tests - see scrapy.tests docstring
 """
 
 if __name__ == '__main__':
-    import os
+    import os, sys
     from twisted.trial import runner, reporter
 
+    tests_to_run = sys.argv[1:] or ['scrapy']
+
     os.environ['SCRAPY_SETTINGS_DISABLED'] = '1'
     loader = runner.TestLoader()
     runner = runner.TrialRunner(reporter.TreeReporter)
-    suite = loader.loadByNames(['scrapy'], recurse=True)
+    suite = loader.loadByNames(tests_to_run, recurse=True)
     runner.run(suite)

scrapy/tests/test_adaptors.py

@@ -1,5 +1,4 @@
 # -*- coding: utf8 -*-
-import os
 import unittest
 import re
 
@@ -7,7 +6,7 @@ from scrapy.item.adaptors import AdaptorPipe
 from scrapy.contrib_exp import adaptors
 from scrapy.http import HtmlResponse, Headers
 from scrapy.xpath.selector import HtmlXPathSelector, XmlXPathSelector
-from scrapy.link import Link
+from scrapy.tests import get_testdata
 
 class AdaptorPipeTestCase(unittest.TestCase):
     def test_pipe_init(self):
@@ -37,12 +36,9 @@ class AdaptorPipeTestCase(unittest.TestCase):
         self.assertTrue(isinstance(pipe1 + sample_callable, AdaptorPipe))
 
 class AdaptorsTestCase(unittest.TestCase):
-    def setUp(self):
-        self.samplesdir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'sample_data', 'adaptors'))
 
     def get_selector(self, domain, url, sample_filename, headers=None, selector=HtmlXPathSelector):
-        sample_filename = os.path.join(self.samplesdir, sample_filename)
-        body = file(sample_filename).read()
+        body = get_testdata('adaptors', sample_filename)
         response = HtmlResponse(url=url, headers=Headers(headers), status=200, body=body)
         return selector(response)
 

scrapy/tests/test_downloadermiddleware_decompression.py

@@ -1,21 +1,14 @@
-import os
 from unittest import TestCase, main
 from scrapy.http import Response, XmlResponse
 from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware
+from scrapy.tests import get_testdata
 
 def setUp():
-    datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'sample_data', 'compressed')
     formats = ['tar', 'xml.bz2', 'xml.gz', 'zip']
-
-    uncompressed_fd = open(os.path.join(datadir, 'feed-sample1.xml'), 'r')
-    uncompressed_body = uncompressed_fd.read()
-    uncompressed_fd.close()
-
+    uncompressed_body = get_testdata('compressed', 'feed-sample1.xml')
     test_responses = {}
     for format in formats:
-        fd = open(os.path.join(datadir, 'feed-sample1.' + format), 'r')
-        body = fd.read()
-        fd.close()
+        body = get_testdata('compressed', 'feed-sample1.' + format)
         test_responses[format] = Response('http://foo.com/bar', body=body)
     return uncompressed_body, test_responses
 

scrapy/tests/test_downloadermiddleware_httpcompression.py

@@ -6,9 +6,10 @@ from os.path import join, abspath, dirname
 from scrapy.spider import spiders
 from scrapy.http import Response, Request
 from scrapy.contrib.downloadermiddleware.httpcompression import HttpCompressionMiddleware
+from scrapy.tests import tests_datadir
 
 
-SAMPLEDIR = join(abspath(dirname(__file__)), 'sample_data/compressed')
+SAMPLEDIR = join(tests_datadir, 'compressed')
 
 FORMAT = {
         'gzip': ('html-gzip.bin', 'gzip'),

scrapy/tests/test_engine.py

@@ -10,6 +10,8 @@ import unittest
 from twisted.internet import reactor
 from twisted.web import server, resource, static, util
 
+from scrapy.tests import tests_datadir
+
 #class TestResource(resource.Resource):
 #    isLeaf = True
 #
@@ -17,7 +19,7 @@ from twisted.web import server, resource, static, util
 #        return "hello world!"
 
 def start_test_site():
-    root_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), "sample_data", "test_site")
+    root_dir = os.path.join(tests_datadir, "test_site")
     r = static.File(root_dir)
 #    r.putChild("test", TestResource())
     r.putChild("redirect", util.Redirect("/redirected"))

scrapy/tests/test_link.py

@@ -1,4 +1,3 @@
-import os
 import re
 import unittest
 
@@ -6,6 +5,7 @@ from scrapy.http import HtmlResponse
 from scrapy.link import LinkExtractor, Link
 from scrapy.link.extractors import RegexLinkExtractor
 from scrapy.contrib.link_extractors import HTMLImageLinkExtractor
+from scrapy.tests import get_testdata
 
 class LinkExtractorTestCase(unittest.TestCase):
     def test_basic(self):
@@ -36,11 +36,10 @@ class LinkExtractorTestCase(unittest.TestCase):
                          [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
 
     def test_extraction_encoding(self):
-        base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
-        body = open(os.path.join(base_path, 'linkextractor_noenc.html'), 'r').read()
+        body = get_testdata('link_extractor', 'linkextractor_noenc.html')
         response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})
         response_noenc = HtmlResponse(url='http://example.com/noenc', body=body)
-        body = open(os.path.join(base_path, 'linkextractor_latin1.html'), 'r').read()
+        body = get_testdata('link_extractor', 'linkextractor_latin1.html')
         response_latin1 = HtmlResponse(url='http://example.com/latin1', body=body)
 
         lx = LinkExtractor()
@@ -66,8 +65,7 @@ class LinkExtractorTestCase(unittest.TestCase):
 
 class RegexLinkExtractorTestCase(unittest.TestCase):
     def setUp(self):
-        base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
-        body = open(os.path.join(base_path, 'regex_linkextractor.html'), 'r').read()
+        body = get_testdata('link_extractor', 'regex_linkextractor.html')
         self.response = HtmlResponse(url='http://example.com/index', body=body)
 
     def test_urls_type(self):
@@ -202,8 +200,7 @@ class RegexLinkExtractorTestCase(unittest.TestCase):
 
 class HTMLImageLinkExtractorTestCase(unittest.TestCase):
     def setUp(self):
-        base_path = os.path.join(os.path.dirname(__file__), 'sample_data', 'link_extractor')
-        body = open(os.path.join(base_path, 'image_linkextractor.html'), 'r').read()
+        body = get_testdata('link_extractor', 'image_linkextractor.html')
         self.response = HtmlResponse(url='http://example.com/index', body=body)
 
     def tearDown(self):

scrapy/tests/test_utils_iterators.py

@@ -4,6 +4,7 @@ import libxml2
 
 from scrapy.utils.iterators import csviter, xmliter
 from scrapy.http import XmlResponse, TextResponse
+from scrapy.tests import get_testdata
 
 class UtilsIteratorsTestCase(unittest.TestCase):
     ### NOTE: Encoding issues have been found with BeautifulSoup for utf-16 files, utf-16 test removed ###
@@ -94,8 +95,7 @@ class UtilsCsvTestCase(unittest.TestCase):
     sample_feed3_path = os.path.join(sample_feeds_dir, 'feed-sample5.csv')
 
     def test_csviter_defaults(self):
-        body = open(self.sample_feed_path).read()
-
+        body = get_testdata('feeds', 'feed-sample3.csv')
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response)
 
@@ -112,8 +112,7 @@ class UtilsCsvTestCase(unittest.TestCase):
             self.assert_(all((isinstance(v, unicode) for v in result_row.values())))
 
     def test_csviter_delimiter(self):
-        body = open(self.sample_feed_path).read().replace(',', '\t')
-
+        body = get_testdata('feeds', 'feed-sample3.csv').replace(',', '\t')
         response = TextResponse(url="http://example.com/", body=body)
         csv = csviter(response, delimiter='\t')
 
@@ -124,7 +123,7 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
     def test_csviter_headers(self):
-        sample = open(self.sample_feed_path).read().splitlines()
+        sample = get_testdata('feeds', 'feed-sample3.csv').splitlines()
         headers, body = sample[0].split(','), '\n'.join(sample[1:])
 
         response = TextResponse(url="http://example.com/", body=body)
@@ -137,7 +136,7 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
     def test_csviter_falserow(self):
-        body = open(self.sample_feed_path).read()
+        body = get_testdata('feeds', 'feed-sample3.csv')
         body = '\n'.join((body, 'a,b', 'a,b,c,d'))
 
         response = TextResponse(url="http://example.com/", body=body)
@@ -150,7 +149,7 @@ class UtilsCsvTestCase(unittest.TestCase):
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
     def test_csviter_exception(self):
-        body = open(self.sample_feed_path).read()
+        body = get_testdata('feeds', 'feed-sample3.csv')
 
         response = TextResponse(url="http://example.com/", body=body)
         iter = csviter(response)
@@ -162,8 +161,8 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertRaises(StopIteration, iter.next)
 
     def test_csviter_encoding(self):
-        body1 = open(self.sample_feed2_path).read()
-        body2 = open(self.sample_feed3_path).read()
+        body1 = get_testdata('feeds', 'feed-sample4.csv')
+        body2 = get_testdata('feeds', 'feed-sample5.csv')
 
         response = TextResponse(url="http://example.com/", body=body1, encoding='latin1')
         csv = csviter(response)