From 94b40162a9f2da77fbc0fc509e8e8be664378e7c Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 30 Aug 2012 11:24:29 -0300 Subject: [PATCH] fixed tests to work on windows --- .../test_downloadermiddleware_decompression.py | 7 ++++--- scrapy/tests/test_utils_iterators.py | 9 +++++---- scrapy/utils/test.py | 6 ++++++ scrapyd/launcher.py | 13 ++++++++++--- scrapyd/tests/test_utils.py | 4 +++- scrapyd/utils.py | 4 +++- 6 files changed, 31 insertions(+), 12 deletions(-) diff --git a/scrapy/tests/test_downloadermiddleware_decompression.py b/scrapy/tests/test_downloadermiddleware_decompression.py index 09dfdbaf8..aaa7d0ab0 100644 --- a/scrapy/tests/test_downloadermiddleware_decompression.py +++ b/scrapy/tests/test_downloadermiddleware_decompression.py @@ -1,8 +1,9 @@ from unittest import TestCase, main -from scrapy.http import Response, XmlResponse, Request +from scrapy.http import Response, XmlResponse from scrapy.contrib_exp.downloadermiddleware.decompression import DecompressionMiddleware from scrapy.spider import BaseSpider from scrapy.tests import get_testdata +from scrapy.utils.test import assert_samelines def _test_data(formats): @@ -29,13 +30,13 @@ class DecompressionMiddlewareTest(TestCase): new = self.mw.process_response(None, rsp, self.spider) assert isinstance(new, XmlResponse), \ 'Failed %s, response type %s' % (fmt, type(new).__name__) - self.assertEqual(new.body, self.uncompressed_body, fmt) + assert_samelines(self, new.body, self.uncompressed_body, fmt) def test_plain_response(self): rsp = Response(url='http://test.com', body=self.uncompressed_body) new = self.mw.process_response(None, rsp, self.spider) assert new is rsp - self.assertEqual(new.body, rsp.body) + assert_samelines(self, new.body, rsp.body) def test_empty_response(self): rsp = Response(url='http://test.com', body='') diff --git a/scrapy/tests/test_utils_iterators.py b/scrapy/tests/test_utils_iterators.py index 06fb4effe..21963031b 100644 --- a/scrapy/tests/test_utils_iterators.py +++ b/scrapy/tests/test_utils_iterators.py @@ -6,6 +6,7 @@ from scrapy.contrib_exp.iterators import xmliter_lxml from scrapy.http import XmlResponse, TextResponse, Response from scrapy.tests import get_testdata +FOOBAR_NL = u"foo" + os.linesep + u"bar" class XmliterTestCase(unittest.TestCase): @@ -142,7 +143,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual(result, [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, + {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, {u'id': u'4', u'name': u'empty', u'value': u''}]) # explicit type check cuz' we no like stinkin' autocasting! yarrr @@ -158,7 +159,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, + {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, {u'id': u'4', u'name': u'empty', u'value': u''}]) def test_csviter_delimiter_binary_response_assume_utf8_encoding(self): @@ -169,7 +170,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, + {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, {u'id': u'4', u'name': u'empty', u'value': u''}]) @@ -196,7 +197,7 @@ class UtilsCsvTestCase(unittest.TestCase): self.assertEqual([row for row in csv], [{u'id': u'1', u'name': u'alpha', u'value': u'foobar'}, {u'id': u'2', u'name': u'unicode', u'value': u'\xfan\xedc\xf3d\xe9\u203d'}, - {u'id': u'3', u'name': u'multi', u'value': u'foo\nbar'}, + {u'id': u'3', u'name': u'multi', u'value': FOOBAR_NL}, {u'id': u'4', u'name': u'empty', u'value': u''}]) def test_csviter_exception(self): diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 42cdb906a..2def768fe 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -66,3 +66,9 @@ def get_pythonpath(): sep = ';' if sys.platform == 'win32' else ':' scrapy_path = __import__('scrapy').__path__[0] return os.path.dirname(scrapy_path) + sep + os.environ.get('PYTHONPATH', '') + +def assert_samelines(testcase, text1, text2, msg=None): + """Asserts text1 and text2 have the same lines, ignoring differences in + line endings between platforms + """ + testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg) diff --git a/scrapyd/launcher.py b/scrapyd/launcher.py index 602554ea9..a283ba5c7 100644 --- a/scrapyd/launcher.py +++ b/scrapyd/launcher.py @@ -18,9 +18,7 @@ class Launcher(Service): self.processes = {} self.finished = [] self.finished_to_keep = config.getint('finished_to_keep', 100) - self.max_proc = config.getint('max_proc', 0) - if not self.max_proc: - self.max_proc = cpu_count() * config.getint('max_proc_per_cpu', 4) + self.max_proc = self._get_max_proc(config) self.runner = config.get('runner', 'scrapyd.runner') self.app = app @@ -55,6 +53,15 @@ class Launcher(Service): del self.finished[:-self.finished_to_keep] # keep last 100 finished jobs self._wait_for_project(slot) + def _get_max_proc(self, config): + max_proc = config.getint('max_proc', 0) + if not max_proc: + try: + cpus = cpu_count() + except NotImplementedError: + cpus = 1 + max_proc = cpus * config.getint('max_proc_per_cpu', 4) + return max_proc class ScrapyProcessProtocol(protocol.ProcessProtocol): diff --git a/scrapyd/tests/test_utils.py b/scrapyd/tests/test_utils.py index 4c3be2fcf..a73c58d2d 100644 --- a/scrapyd/tests/test_utils.py +++ b/scrapyd/tests/test_utils.py @@ -4,6 +4,7 @@ from cStringIO import StringIO from twisted.trial import unittest +from scrapy.utils.test import get_pythonpath from scrapyd.interfaces import IEggStorage from scrapyd.utils import get_crawl_args, get_spider_list from scrapyd import get_application @@ -45,5 +46,6 @@ class GetSpiderListTest(unittest.TestCase): eggstorage = app.getComponent(IEggStorage) eggfile = StringIO(get_data(__package__, 'mybot.egg')) eggstorage.put(eggfile, 'mybot', 'r1') - self.assertEqual(sorted(get_spider_list('mybot')), ['spider1', 'spider2']) + spiders = get_spider_list('mybot', pythonpath=get_pythonpath()) + self.assertEqual(sorted(spiders), ['spider1', 'spider2']) diff --git a/scrapyd/utils.py b/scrapyd/utils.py index 4ec5b94da..a299ba6a9 100644 --- a/scrapyd/utils.py +++ b/scrapyd/utils.py @@ -49,12 +49,14 @@ def get_crawl_args(message): args += ['%s=%s' % (k, v)] return args -def get_spider_list(project, runner=None): +def get_spider_list(project, runner=None, pythonpath=None): """Return the spider list from the given project, using the given runner""" if runner is None: runner = Config().get('runner') env = os.environ.copy() env['SCRAPY_PROJECT'] = project + if pythonpath: + env['PYTHONPATH'] = pythonpath pargs = [sys.executable, '-m', runner, 'list'] proc = Popen(pargs, stdout=PIPE, stderr=PIPE, env=env) out, err = proc.communicate()