From 9c256cf693d73e854d409d717854b3f354b5e0a9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Mar 2017 15:41:57 -0300 Subject: [PATCH] Undocument _InvalidOutput exception --- docs/topics/exceptions.rst | 11 ----------- scrapy/core/spidermw.py | 12 ++++++------ scrapy/exceptions.py | 8 +++++--- tests/test_spidermiddleware.py | 12 ++++++------ 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index a3ff72827..cc02369d4 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -62,17 +62,6 @@ remain disabled. Those components include: The exception must be raised in the component's ``__init__`` method. -InvalidOutput -------------- - -.. exception:: InvalidOutput - -This exception can be raised by a downloader or spider middleware to -indicate that some method returned a value not suported by the processing -chain. -See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware` -for a list of supported output values. - NotSupported ------------ diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0f03a7b36..50677670b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,7 +5,7 @@ See documentation in docs/topics/spider-middleware.rst """ import six from twisted.python.failure import Failure -from scrapy.exceptions import InvalidOutput +from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list @@ -42,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - raise InvalidOutput('Middleware {} must return None or raise ' \ + raise _InvalidOutput('Middleware {} must return None or raise ' \ 'an exception, got {}'.format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) @@ -50,13 +50,13 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_spider_exception(_failure): exception = _failure.value - # don't handle InvalidOutput exception - if isinstance(exception, InvalidOutput): + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): return _failure for method in self.methods['process_spider_exception']: result = method(response=response, exception=exception, spider=spider) if result is not None and not _isiterable(result): - raise InvalidOutput('Middleware {} must return None or an iterable ' \ + raise _InvalidOutput('Middleware {} must return None or an iterable ' \ 'object, got {}'.format(fname(method), type(result))) # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned @@ -80,7 +80,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = wrapper(result) else: - raise InvalidOutput('Middleware {} must return an iterable object, ' \ + raise _InvalidOutput('Middleware {} must return an iterable object, ' \ 'got {}'.format(fname(method), type(result))) return result diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index ba7272255..96949bdd9 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -11,9 +11,11 @@ class NotConfigured(Exception): """Indicates a missing configuration situation""" pass -class InvalidOutput(TypeError): - """Indicates an invalid value has been returned - by a middleware's processing method""" +class _InvalidOutput(TypeError): + """ + Indicates an invalid value has been returned by a middleware's processing method. + Internal and undocumented, it should not be raised or caught by user code. + """ pass # HTTP and crawling diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 3981a8d75..2d05c335c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -159,7 +159,7 @@ class DoSomethingMiddleware(object): # ================================================================================ -# don't catch InvalidOutput from scrapy's spider middleware manager +# don't catch _InvalidOutput from scrapy's spider middleware manager class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_input' custom_settings = { @@ -318,19 +318,19 @@ class TestSpiderMiddleware(TestCase): @defer.inlineCallbacks def test_process_spider_exception_invalid_return_value_previous_middleware(self): - """ don't catch InvalidOutput from middleware """ + """ don't catch _InvalidOutput from middleware """ # on middleware's input crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) with LogCapture() as log1: yield crawler1.crawl() - self.assertNotIn("InvalidOutput exception caught", str(log1)) - self.assertIn("'spider_exceptions/InvalidOutput'", str(log1)) + self.assertNotIn("_InvalidOutput exception caught", str(log1)) + self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) # on middleware's output crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) with LogCapture() as log2: yield crawler2.crawl() - self.assertNotIn("InvalidOutput exception caught", str(log2)) - self.assertIn("'spider_exceptions/InvalidOutput'", str(log2)) + self.assertNotIn("_InvalidOutput exception caught", str(log2)) + self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) @defer.inlineCallbacks def test_process_spider_exception_execution_chain(self):