fixed tests to work on windows

scrapy/tests/test_downloadermiddleware_decompression.py

@@ -1,8 +1,9 @@
 from unittest import TestCase, main
-from scrapy.http import Response, XmlResponse, Request
+from scrapy.http import Response, XmlResponse
 from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware
 from scrapy.spider import BaseSpider
 from scrapy.tests import get_testdata
+from scrapy.utils.test import assert_samelines
 
 
 def _test_data(formats):
@@ -29,13 +30,13 @@ class DecompressionMiddlewareTest(TestCase):
             new = self.mw.process_response(None, rsp, self.spider)
             assert isinstance(new, XmlResponse), \
                     'Failed %s, response type %s' % (fmt, type(new).__name__)
-            self.assertEqual(new.body, self.uncompressed_body, fmt)
+            assert_samelines(self, new.body, self.uncompressed_body, fmt)
 
     def test_plain_response(self):
         rsp = Response(url='http://test.com', body=self.uncompressed_body)
         new = self.mw.process_response(None, rsp, self.spider)
         assert new is rsp
-        self.assertEqual(new.body, rsp.body)
+        assert_samelines(self, new.body, rsp.body)
 
     def test_empty_response(self):
         rsp = Response(url='http://test.com', body='')

scrapy/tests/test_utils_iterators.py

@@ -6,6 +6,7 @@ from scrapy.contrib_exp.iterators import xmliter_lxml
 from scrapy.http import XmlResponse, TextResponse, Response
 from scrapy.tests import get_testdata
 
+FOOBAR_NL = u"foo" + os.linesep + u"bar"
 
 class XmliterTestCase(unittest.TestCase):
 
@@ -142,7 +143,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual(result,
                          [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
+                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
         # explicit type check cuz' we no like stinkin' autocasting! yarrr
@@ -158,7 +159,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv],
                          [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
+                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
     def test_csviter_delimiter_binary_response_assume_utf8_encoding(self):
@@ -169,7 +170,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv],
                          [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
+                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
 
@@ -196,7 +197,7 @@ class UtilsCsvTestCase(unittest.TestCase):
         self.assertEqual([row for row in csv],
                          [{u'id': u'1', u'name': u'alpha',   u'value': u'foobar'},
                           {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'},
-                          {u'id': u'3', u'name': u'multi',   u'value': u'foo\nbar'},
+                          {u'id': u'3', u'name': u'multi',   u'value': FOOBAR_NL},
                           {u'id': u'4', u'name': u'empty',   u'value': u''}])
 
     def test_csviter_exception(self):

scrapy/utils/test.py

@@ -66,3 +66,9 @@ def get_pythonpath():
     sep = ';' if sys.platform == 'win32' else ':'
     scrapy_path = __import__('scrapy').__path__[0]
     return os.path.dirname(scrapy_path) + sep + os.environ.get('PYTHONPATH', '')
+
+def assert_samelines(testcase, text1, text2, msg=None):
+    """Asserts text1 and text2 have the same lines, ignoring differences in
+    line endings between platforms
+    """
+    testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)

scrapyd/launcher.py

@@ -18,9 +18,7 @@ class Launcher(Service):
         self.processes = {}
         self.finished = []
         self.finished_to_keep = config.getint('finished_to_keep', 100)
-        self.max_proc = config.getint('max_proc', 0)
-        if not self.max_proc:
-            self.max_proc = cpu_count() * config.getint('max_proc_per_cpu', 4)
+        self.max_proc = self._get_max_proc(config)
         self.runner = config.get('runner', 'scrapyd.runner')
         self.app = app
 
@@ -55,6 +53,15 @@ class Launcher(Service):
         del self.finished[:-self.finished_to_keep] # keep last 100 finished jobs
         self._wait_for_project(slot)
 
+    def _get_max_proc(self, config):
+        max_proc = config.getint('max_proc', 0)
+        if not max_proc:
+            try:
+                cpus = cpu_count()
+            except NotImplementedError:
+                cpus = 1
+            max_proc = cpus * config.getint('max_proc_per_cpu', 4)
+        return max_proc
 
 class ScrapyProcessProtocol(protocol.ProcessProtocol):
 

scrapyd/tests/test_utils.py

@@ -4,6 +4,7 @@ from cStringIO import StringIO
 
 from twisted.trial import unittest
 
+from scrapy.utils.test import get_pythonpath
 from scrapyd.interfaces import IEggStorage
 from scrapyd.utils import get_crawl_args, get_spider_list
 from scrapyd import get_application
@@ -45,5 +46,6 @@ class GetSpiderListTest(unittest.TestCase):
         eggstorage = app.getComponent(IEggStorage)
         eggfile = StringIO(get_data(__package__, 'mybot.egg'))
         eggstorage.put(eggfile, 'mybot', 'r1')
-        self.assertEqual(sorted(get_spider_list('mybot')), ['spider1', 'spider2'])
+        spiders = get_spider_list('mybot', pythonpath=get_pythonpath())
+        self.assertEqual(sorted(spiders), ['spider1', 'spider2'])
 

scrapyd/utils.py

@@ -49,12 +49,14 @@ def get_crawl_args(message):
         args += ['%s=%s' % (k, v)]
     return args
 
-def get_spider_list(project, runner=None):
+def get_spider_list(project, runner=None, pythonpath=None):
     """Return the spider list from the given project, using the given runner"""
     if runner is None:
         runner = Config().get('runner')
     env = os.environ.copy()
     env['SCRAPY_PROJECT'] = project
+    if pythonpath:
+        env['PYTHONPATH'] = pythonpath
     pargs = [sys.executable, '-m', runner, 'list']
     proc = Popen(pargs, stdout=PIPE, stderr=PIPE, env=env)
     out, err = proc.communicate()