From 06da7af9e293d00087799ac819f9ac4caa4901e7 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 17 Feb 2016 23:03:12 +0500 Subject: [PATCH 1/4] TST clean up RunSpiderCommandTest --- tests/test_commands.py | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 1a30368ba..5d45cd62d 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -6,7 +6,7 @@ from time import sleep from os.path import exists, join, abspath from shutil import rmtree, copytree from tempfile import mkdtemp -import six +from contextlib import contextmanager from twisted.trial import unittest from twisted.internet import defer @@ -154,12 +154,24 @@ class MiscCommandsTest(CommandTest): class RunSpiderCommandTest(CommandTest): - def test_runspider(self): + @contextmanager + def _create_file(self, content, name): tmpdir = self.mktemp() os.mkdir(tmpdir) - fname = abspath(join(tmpdir, 'myspider.py')) + fname = abspath(join(tmpdir, name)) with open(fname, 'w') as f: - f.write(""" + f.write(content) + try: + yield fname + finally: + rmtree(tmpdir) + + def runspider(self, code, name='myspider.py'): + with self._create_file(code, name) as fname: + return self.proc('runspider', fname) + + def test_runspider(self): + spider = """ import scrapy class MySpider(scrapy.Spider): @@ -168,23 +180,17 @@ class MySpider(scrapy.Spider): def start_requests(self): self.logger.debug("It Works!") return [] -""") - p = self.proc('runspider', fname) +""" + p = self.runspider(spider) log = to_native_str(p.stderr.read()) + self.assertIn("DEBUG: It Works!", log) self.assertIn("INFO: Spider opened", log) self.assertIn("INFO: Closing spider (finished)", log) self.assertIn("INFO: Spider closed (finished)", log) def test_runspider_no_spider_found(self): - tmpdir = self.mktemp() - os.mkdir(tmpdir) - fname = abspath(join(tmpdir, 'myspider.py')) - with open(fname, 'w') as f: - f.write(""" -from scrapy.spiders import Spider -""") - p = self.proc('runspider', fname) + p = self.runspider("from scrapy.spiders import Spider\n") log = to_native_str(p.stderr.read()) self.assertIn("No spider found in file", log) @@ -194,12 +200,7 @@ from scrapy.spiders import Spider self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): - tmpdir = self.mktemp() - os.mkdir(tmpdir) - fname = abspath(join(tmpdir, 'myspider.txt')) - with open(fname, 'w') as f: - f.write("") - p = self.proc('runspider', fname) + p = self.runspider("", "myspider.txt") log = to_native_str(p.stderr.read()) self.assertIn("Unable to load", log) From f766dd0ba8afce0ec9890106fe0fd477aaf04d29 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 17 Feb 2016 23:07:03 +0500 Subject: [PATCH 2/4] Preserve tracebacks better. Fixes GH-1760. --- scrapy/crawler.py | 15 +++++++++++++-- tests/test_commands.py | 14 ++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ef99c243a..3e695f793 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -3,6 +3,7 @@ import signal import logging import warnings +import sys from twisted.internet import reactor, defer from zope.interface.verify import verifyClass, DoesNotImplement @@ -73,11 +74,21 @@ class Crawler(object): yield self.engine.open_spider(self.spider, start_requests) yield defer.maybeDeferred(self.engine.start) except Exception: - exc = defer.fail() + # In Python 2 reraising an exception after yield discards + # the original traceback (see http://bugs.python.org/issue7563), + # so sys.exc_info() workaround is used. + # This workaround also works in Python 3, but it is not needed, + # and it is slower, so in Python 3 we use native `raise`. + if six.PY2: + exc_info = sys.exc_info() + self.crawling = False if self.engine is not None: yield self.engine.close() - yield exc + + if six.PY2: + raise six.reraise(*exc_info) + raise def _create_spider(self, *args, **kwargs): return self.spidercls.from_crawler(self, *args, **kwargs) diff --git a/tests/test_commands.py b/tests/test_commands.py index 5d45cd62d..93b53dbeb 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -204,6 +204,20 @@ class MySpider(scrapy.Spider): log = to_native_str(p.stderr.read()) self.assertIn("Unable to load", log) + def test_start_requests_errors(self): + p = self.runspider(""" +import scrapy + +class BadSpider(scrapy.Spider): + name = "bad" + def start_requests(self): + raise Exception("oops!") + """, name="badspider.py") + log = to_native_str(p.stderr.read()) + print(log) + self.assertIn("start_requests", log) + self.assertIn("badspider.py", log) + class ParseCommandTest(ProcessTest, SiteTest, CommandTest): command = 'parse' From 104027d78d10592a395ffeb8f34ac9fc0ae1702f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 18 Feb 2016 11:45:03 +0100 Subject: [PATCH 3/4] Minor change on quotes Trying to force Travis CI to build --- tests/test_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_commands.py b/tests/test_commands.py index 93b53dbeb..2e47160d7 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -200,9 +200,9 @@ class MySpider(scrapy.Spider): self.assertIn("File not found: some_non_existent_file", log) def test_runspider_unable_to_load(self): - p = self.runspider("", "myspider.txt") + p = self.runspider('', 'myspider.txt') log = to_native_str(p.stderr.read()) - self.assertIn("Unable to load", log) + self.assertIn('Unable to load', log) def test_start_requests_errors(self): p = self.runspider(""" From 18a381816d1e2e4a08e8c8655fa38f9bb0af312f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 19 Feb 2016 18:31:49 +0500 Subject: [PATCH 4/4] Remove unneeded `raise` (thanks @lopuhin). --- scrapy/crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 3e695f793..443a9aa2f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -87,7 +87,7 @@ class Crawler(object): yield self.engine.close() if six.PY2: - raise six.reraise(*exc_info) + six.reraise(*exc_info) raise def _create_spider(self, *args, **kwargs):