From 986be9a3965f4fef071beb0f726e30a700d9e227 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 28 Mar 2014 13:53:53 -0300 Subject: [PATCH 1/8] Run testsuite with py.test --- .travis.yml | 1 + tests-requirements.txt | 1 + tox.ini | 12 ++++++++---- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 367a56280..2f2092a74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,7 @@ env: - TOXENV=precise - TOXENV=trunk - TOXENV=pypy +- TOXENV=py33 matrix: allow_failures: - env: TOXENV=pypy diff --git a/tests-requirements.txt b/tests-requirements.txt index 34275dfa7..b7d6a0a56 100644 --- a/tests-requirements.txt +++ b/tests-requirements.txt @@ -1,3 +1,4 @@ # Tests requirements mock mitmproxy >= 0.10 +pytest-twisted diff --git a/tox.ini b/tox.ini index cebffbccc..597abfb5d 100644 --- a/tox.ini +++ b/tox.ini @@ -5,6 +5,8 @@ [tox] envlist = py27, pypy, precise, trunk, py33 +indexserver = + HPK = https://devpi.net/hpk/dev/ [testenv] deps = @@ -15,7 +17,7 @@ deps = django -rtests-requirements.txt commands = - trial {posargs:scrapy} + py.test --twisted {posargs:scrapy} [testenv:precise] basepython = python2.7 @@ -34,7 +36,7 @@ basepython = python2.7 commands = pip install https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib pip install https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib - trial {posargs:scrapy} + py.test --twisted {posargs:scrapy} [testenv:py33] basepython = python3.3 @@ -45,8 +47,10 @@ deps = cssselect>=0.9 queuelib>=1.1.1 w3lib>=1.5 -commands = - trial {posargs:scrapy} + # tests requirements + mock + :HPK:pytest>2.5.2 + pytest-twisted [testenv:windows] commands = From 20b4c8f3eac1ed9e5b48c6b19c65b405d3646bb7 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 4 Apr 2014 16:38:09 -0300 Subject: [PATCH 2/8] Trial functionality for running tests with pytest * Change current dir to tmp dir on each test run * Log twisted with test.log --- pytest.ini | 3 +++ scrapy/conftest.py | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 pytest.ini create mode 100644 scrapy/conftest.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..b845effc9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ + +[pytest] +usefixtures = chdir setlog diff --git a/scrapy/conftest.py b/scrapy/conftest.py new file mode 100644 index 000000000..e98e9076f --- /dev/null +++ b/scrapy/conftest.py @@ -0,0 +1,39 @@ + +import pytest +from twisted.python import log + + +class LogObservers: + """Class for keeping track of log observers across test modules""" + + def __init__(self): + self.observers = [] + + def add(self, logfile='test.log'): + fileobj = open(logfile, 'wb') + observer = log.FileLogObserver(fileobj) + log.startLoggingWithObserver(observer.emit, 0) + self.observers.append((fileobj, observer)) + + def remove(self): + fileobj, observer = self.observers.pop() + log.removeObserver(observer.emit) + fileobj.close() + + +@pytest.fixture(scope='module') +def log_observers(): + return LogObservers() + + +@pytest.fixture() +def setlog(request, log_observers): + """Attach test.log file observer to twisted log, for trial compatibility""" + log_observers.add() + request.addfinalizer(log_observers.remove) + + +@pytest.fixture() +def chdir(tmpdir): + """Change to pytest-provided temporary directory""" + tmpdir.chdir() From 1779b31e9b631f8f077a0523affbb85b2056c8b2 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 3 Apr 2014 13:14:59 -0300 Subject: [PATCH 3/8] Add py33 environment to allowed failures in travis-ci --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 2f2092a74..08ca870a5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ env: matrix: allow_failures: - env: TOXENV=pypy + - env: TOXENV=py33 install: - ./.travis-workarounds.sh - pip install tox From 560a84657ca4e19086fabbe34ec0ab73e6220139 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 4 Apr 2014 16:54:40 -0300 Subject: [PATCH 4/8] Support doctest and __init__.py test discover in pytest --- pytest.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest.ini b/pytest.ini index b845effc9..8d3bc91a5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,5 @@ [pytest] usefixtures = chdir setlog +python_files=test_*.py __init__.py +addopts = --doctest-modules From 81187feaac0647e5ec40781f03326ce5d73378b4 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 4 Apr 2014 16:55:35 -0300 Subject: [PATCH 5/8] Ignore files with import errors on pytest test discover --- scrapy/conftest.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scrapy/conftest.py b/scrapy/conftest.py index e98e9076f..7e11f331f 100644 --- a/scrapy/conftest.py +++ b/scrapy/conftest.py @@ -2,6 +2,12 @@ import pytest from twisted.python import log +from scrapy import optional_features + +collect_ignore = ["stats.py"] +if 'django' not in optional_features: + collect_ignore.append("tests/test_djangoitem/models.py") + class LogObservers: """Class for keeping track of log observers across test modules""" From 118696971538fa2bf2388f8ace525ca5158ee545 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Thu, 3 Apr 2014 05:58:15 -0300 Subject: [PATCH 6/8] Change function name so it does not mess up with pytest autodiscover --- scrapy/tests/test_squeue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_squeue.py b/scrapy/tests/test_squeue.py index a3c64db8d..83ffcc4b7 100644 --- a/scrapy/tests/test_squeue.py +++ b/scrapy/tests/test_squeue.py @@ -7,12 +7,12 @@ from scrapy.contrib.loader import ItemLoader class TestItem(Item): name = Field() -def test_processor(x): +def _test_procesor(x): return x + x class TestLoader(ItemLoader): default_item_class = TestItem - name_out = staticmethod(test_processor) + name_out = staticmethod(_test_procesor) class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest): From c50f088b631d259fd401f6a100acfa8bba9bf68d Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 4 Apr 2014 16:58:08 -0300 Subject: [PATCH 7/8] Fix httpcache doctest that assumed dictionary order --- scrapy/contrib/httpcache.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 5dee898f0..fefebc41c 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -290,10 +290,11 @@ def parse_cachecontrol(header): http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 - >>> parse_cachecontrol('public, max-age=3600') - {'public': None, 'max-age': '3600'} - >>> parse_cachecontrol('') - {} + >>> parse_cachecontrol('public, max-age=3600') == {'public': None, + ... 'max-age': '3600'} + True + >>> parse_cachecontrol('') == {} + True """ directives = {} From dd9d3af9b3a478fb17e8fa721ee901ea68a3e833 Mon Sep 17 00:00:00 2001 From: Julia Medina Date: Fri, 4 Apr 2014 17:01:42 -0300 Subject: [PATCH 8/8] Ensure spiders module reload between spider manager tests --- scrapy/tests/test_spidermanager/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/tests/test_spidermanager/__init__.py b/scrapy/tests/test_spidermanager/__init__.py index c8697d3b5..eebd6b298 100644 --- a/scrapy/tests/test_spidermanager/__init__.py +++ b/scrapy/tests/test_spidermanager/__init__.py @@ -27,6 +27,7 @@ class SpiderManagerTest(unittest.TestCase): def tearDown(self): del self.spiderman + del sys.modules['test_spiders_xxx'] sys.path.remove(self.tmpdir) def test_interface(self):