From 706ed0e049fe008cac12f243371b67ee0230a08a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 14 Jun 2016 12:34:07 -0300 Subject: [PATCH 001/140] Spider middleware: process_spider_exception on generators --- docs/topics/exceptions.rst | 11 + docs/topics/spider-middleware.rst | 10 +- scrapy/core/spidermw.py | 37 +++- scrapy/exceptions.py | 5 + tests/test_spidermiddleware.py | 340 ++++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+), 14 deletions(-) create mode 100644 tests/test_spidermiddleware.py diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index cc02369d4..a3ff72827 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -62,6 +62,17 @@ remain disabled. Those components include: The exception must be raised in the component's ``__init__`` method. +InvalidOutput +------------- + +.. exception:: InvalidOutput + +This exception can be raised by a downloader or spider middleware to +indicate that some method returned a value not suported by the processing +chain. +See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware` +for a list of supported output values. + NotSupported ------------ diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8360827e8..fc7669437 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -112,11 +112,12 @@ following methods: .. method:: process_spider_exception(response, exception, spider) - This method is called when a spider or :meth:`process_spider_input` - method (from other spider middleware) raises an exception. + This method is called when when a spider or :meth:`process_spider_input`/ + :meth:`process_spider_output` method (from other spider middleware) + raises an exception. :meth:`process_spider_exception` should return either ``None`` or an - iterable of :class:`~scrapy.http.Response`, dict or + iterable of :class:`~scrapy.http.Request`, dict or :class:`~scrapy.item.Item` objects. If it returns ``None``, Scrapy will continue processing this exception, @@ -125,7 +126,8 @@ following methods: exception reaches the engine (where it's logged and discarded). If it returns an iterable the :meth:`process_spider_output` pipeline - kicks in, and no other :meth:`process_spider_exception` will be called. + kicks in, starting with the last non-executed method, and no other + :meth:`process_spider_exception` will be called. :param response: the response being processed when the exception was raised diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index a206e4b0c..0f03a7b36 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,6 +5,7 @@ See documentation in docs/topics/spider-middleware.rst """ import six from twisted.python.failure import Failure +from scrapy.exceptions import InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list @@ -40,31 +41,47 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods['process_spider_input']: try: result = method(response=response, spider=spider) - assert result is None, \ - 'Middleware %s must returns None or ' \ - 'raise an exception, got %s ' \ - % (fname(method), type(result)) + if result is not None: + raise InvalidOutput('Middleware {} must return None or raise ' \ + 'an exception, got {}'.format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) def process_spider_exception(_failure): exception = _failure.value + # don't handle InvalidOutput exception + if isinstance(exception, InvalidOutput): + return _failure for method in self.methods['process_spider_exception']: result = method(response=response, exception=exception, spider=spider) - assert result is None or _isiterable(result), \ - 'Middleware %s must returns None, or an iterable object, got %s ' % \ - (fname(method), type(result)) + if result is not None and not _isiterable(result): + raise InvalidOutput('Middleware {} must return None or an iterable ' \ + 'object, got {}'.format(fname(method), type(result))) + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned if result is not None: return result return _failure def process_spider_output(result): + def wrapper(result_iterable): + try: + for r in result_iterable: + yield r + except Exception as ex: + exception_result = process_spider_exception(Failure(ex)) + if exception_result is None or isinstance(exception_result, Failure): + raise + for output in exception_result: + yield output for method in self.methods['process_spider_output']: result = method(response=response, result=result, spider=spider) - assert _isiterable(result), \ - 'Middleware %s must returns an iterable object, got %s ' % \ - (fname(method), type(result)) + if _isiterable(result): + result = wrapper(result) + else: + raise InvalidOutput('Middleware {} must return an iterable object, ' \ + 'got {}'.format(fname(method), type(result))) return result dfd = mustbe_deferred(process_spider_input, response) diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index 4bcecd994..ba7272255 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -11,6 +11,11 @@ class NotConfigured(Exception): """Indicates a missing configuration situation""" pass +class InvalidOutput(TypeError): + """Indicates an invalid value has been returned + by a middleware's processing method""" + pass + # HTTP and crawling class IgnoreRequest(Exception): diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py new file mode 100644 index 000000000..8ec9583d8 --- /dev/null +++ b/tests/test_spidermiddleware.py @@ -0,0 +1,340 @@ + +import logging + +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase +from twisted.internet import defer + +from scrapy.spiders import Spider +from scrapy.item import Item, Field +from scrapy.http import Request +from scrapy.utils.test import get_crawler + + +class TestItem(Item): + value = Field() + + +# ================================================================================ +# exceptions from a spider's parse method +class BaseExceptionFromParseMethodSpider(Spider): + start_urls = ["http://example.com/"] + custom_settings = { + 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540} + } + + +class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider): + """ return value is NOT a generator """ + name = 'not_a_generator' + + def parse(self, response): + raise AssertionError + + +class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider): + """ return value is a generator; the exception is raised + before the items are yielded: no items should be scraped """ + name = 'generator_error_before_items' + + def parse(self, response): + raise ValueError + for i in range(3): + yield {'value': i} + + +class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider): + """ return value is a generator; the exception is raised + after the items are yielded: 3 items should be scraped """ + name = 'generator_error_after_items' + + def parse(self, response): + for i in range(3): + yield {'value': i} + raise FloatingPointError + + +class CatchExceptionMiddleware(object): + def process_spider_exception(self, response, exception, spider): + """ catch an exception and log it """ + logging.warn('{} exception caught'.format(exception.__class__.__name__)) + return None + + +# ================================================================================ +# exception from a previous middleware's process_spider_input method +# process_spider_input is not expected to return an iterable, so there are no +# separate tests for generator/non-generator implementations +class FromPreviousMiddlewareInputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'not_a_generator_from_previous_middleware_input' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return None + + +class RaiseExceptionOnInputMiddleware(object): + def process_spider_input(self, response, spider): + raise LookupError + + +# ================================================================================ +# exception from a previous middleware's process_spider_output method (not a generator) +class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'not_a_generator_from_previous_middleware_output' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return [{'value': i} for i in range(3)] + + +class RaiseExceptionOnOutputNotAGeneratorMiddleware(object): + def process_spider_output(self, response, result, spider): + raise UnicodeError + + +# ================================================================================ +# exception from a previous middleware's process_spider_output method (generator) +class GeneratorFromPreviousMiddlewareOutputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'generator_from_previous_middleware_output' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return [{'value': i} for i in range(10, 13)] + + +class RaiseExceptionOnOutputGeneratorMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + yield r + raise NameError + + +# ================================================================================ +# do something useful from the exception handler +class DoSomethingSpider(Spider): + start_urls = ["http://example.com"] + name = 'do_something' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.DoSomethingMiddleware': 540, + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, + # spider side + } + } + + def parse(self, response): + yield {'value': response.url} + raise ImportError + + +class DoSomethingMiddleware(object): + def process_spider_exception(self, response, exception, spider): + return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')] + + +# ================================================================================ +# don't catch InvalidOutput from scrapy's spider middleware manager +class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'invalid_return_value_from_previous_middleware_input' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540, + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return None + + +class InvalidReturnValueInputMiddleware(object): + def process_spider_input(self, response, spider): + return 1.0 # , not None + + +class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider): + start_urls = ["http://example.com/"] + name = 'invalid_return_value_from_previous_middleware_output' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, + 'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545, + # spider side + } + } + + def parse(self, response): + return None + + +class InvalidReturnValueOutputMiddleware(object): + def process_spider_output(self, response, result, spider): + return 1 # , not an iterable + + +# ================================================================================ +# make sure only non already called process_spider_output methods +# are called if process_spider_exception returns an iterable +class ExecutionChainSpider(Spider): + start_urls = ["http://example.com"] + name = 'execution_chain' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # engine side + 'tests.test_spidermiddleware.ThirdMiddleware': 540, + 'tests.test_spidermiddleware.SecondMiddleware': 541, + 'tests.test_spidermiddleware.FirstMiddleware': 542 + # spider side + }, + } + + def parse(self, response): + return None + + +class FirstMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + if isinstance(r, dict): + r['handled_by_first_middleware'] = True + yield r + + def process_spider_exception(self, response, exception, spider): + # log exception, handle control to the next middleware's process_spider_exception + logging.warn('{} exception caught'.format(exception.__class__.__name__)) + return None + + +class SecondMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + if isinstance(r, dict): + r['handled_by_second_middleware'] = True + yield r + raise MemoryError + + +class ThirdMiddleware(object): + def process_spider_output(self, response, result, spider): + for r in result: + if isinstance(r, dict): + r['handled_by_third_middleware'] = True + yield r + + def process_spider_exception(self, response, exception, spider): + # handle control to the next middleware's process_spider_output + return [{'item': i} for i in range(3)] + + +class TestSpiderMiddleware(TestCase): + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method(self): + # non-generator return value + crawler = get_crawler(NotAGeneratorSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("AssertionError exception caught", str(log)) + self.assertIn("spider_exceptions/AssertionError", str(log)) + # generator return value, no items before the error + crawler = get_crawler(GeneratorErrorBeforeItemsSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("ValueError exception caught", str(log)) + self.assertIn("spider_exceptions/ValueError", str(log)) + # generator return value, 3 items before the error + crawler = get_crawler(GeneratorErrorAfterItemsSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("'item_scraped_count': 3", str(log)) + self.assertIn("FloatingPointError exception caught", str(log)) + self.assertIn("spider_exceptions/FloatingPointError", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_previous_middleware_input(self): + crawler = get_crawler(FromPreviousMiddlewareInputSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("LookupError exception caught", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_previous_middleware_output(self): + # non-generator output value + crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertNotIn("UnicodeError exception caught", str(log)) + # generator output value + crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("'item_scraped_count': 3", str(log)) + self.assertIn("NameError exception caught", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_do_something(self): + crawler = get_crawler(DoSomethingSpider) + with LogCapture() as log: + yield crawler.crawl() + self.assertIn("ImportError exception caught", str(log)) + self.assertIn("{'value': 10}", str(log)) + self.assertIn("{'value': 'asdf'}", str(log)) + self.assertIn("{'value': 'http://example.com'}", str(log)) + self.assertIn("{'value': 'http://example.org'}", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_invalid_return_value_previous_middleware(self): + """ don't catch InvalidOutput from middleware """ + # on middleware's input + crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) + with LogCapture() as log1: + yield crawler1.crawl() + self.assertNotIn("InvalidOutput exception caught", str(log1)) + self.assertIn("'spider_exceptions/InvalidOutput'", str(log1)) + # on middleware's output + crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) + with LogCapture() as log2: + yield crawler2.crawl() + self.assertNotIn("InvalidOutput exception caught", str(log2)) + self.assertIn("'spider_exceptions/InvalidOutput'", str(log2)) + + @defer.inlineCallbacks + def test_process_spider_exception_execution_chain(self): + # on middleware's input + crawler1 = get_crawler(ExecutionChainSpider) + with LogCapture() as log1: + yield crawler1.crawl() + self.assertNotIn("handled_by_first_middleware", str(log1)) + self.assertNotIn("handled_by_second_middleware", str(log1)) + self.assertIn("MemoryError exception caught", str(log1)) + self.assertIn("handled_by_third_middleware", str(log1)) From 4090cc3990636337964a6e157679d6be15ba6f3a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 8 Mar 2017 18:11:20 -0300 Subject: [PATCH 002/140] Spider middleware: use Mockserver to test process_spider_exception --- tests/test_spidermiddleware.py | 42 +++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 8ec9583d8..3981a8d75 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -9,16 +9,20 @@ from scrapy.spiders import Spider from scrapy.item import Item, Field from scrapy.http import Request from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer class TestItem(Item): value = Field() +class LocalhostSpider(Spider): + start_urls = ['http://localhost:8998'] # tests.mockserver.MockServer + + # ================================================================================ # exceptions from a spider's parse method -class BaseExceptionFromParseMethodSpider(Spider): - start_urls = ["http://example.com/"] +class BaseExceptionFromParseMethodSpider(LocalhostSpider): custom_settings = { 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540} } @@ -65,8 +69,7 @@ class CatchExceptionMiddleware(object): # exception from a previous middleware's process_spider_input method # process_spider_input is not expected to return an iterable, so there are no # separate tests for generator/non-generator implementations -class FromPreviousMiddlewareInputSpider(Spider): - start_urls = ["http://example.com/"] +class FromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'not_a_generator_from_previous_middleware_input' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -88,8 +91,7 @@ class RaiseExceptionOnInputMiddleware(object): # ================================================================================ # exception from a previous middleware's process_spider_output method (not a generator) -class NotAGeneratorFromPreviousMiddlewareOutputSpider(Spider): - start_urls = ["http://example.com/"] +class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): name = 'not_a_generator_from_previous_middleware_output' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -111,8 +113,7 @@ class RaiseExceptionOnOutputNotAGeneratorMiddleware(object): # ================================================================================ # exception from a previous middleware's process_spider_output method (generator) -class GeneratorFromPreviousMiddlewareOutputSpider(Spider): - start_urls = ["http://example.com/"] +class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): name = 'generator_from_previous_middleware_output' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -136,8 +137,7 @@ class RaiseExceptionOnOutputGeneratorMiddleware(object): # ================================================================================ # do something useful from the exception handler -class DoSomethingSpider(Spider): - start_urls = ["http://example.com"] +class DoSomethingSpider(LocalhostSpider): name = 'do_something' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -155,13 +155,12 @@ class DoSomethingSpider(Spider): class DoSomethingMiddleware(object): def process_spider_exception(self, response, exception, spider): - return [Request('http://example.org'), {'value': 10}, TestItem(value='asdf')] + return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')] # ================================================================================ # don't catch InvalidOutput from scrapy's spider middleware manager -class InvalidReturnValueFromPreviousMiddlewareInputSpider(Spider): - start_urls = ["http://example.com/"] +class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_input' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -181,8 +180,7 @@ class InvalidReturnValueInputMiddleware(object): return 1.0 # , not None -class InvalidReturnValueFromPreviousMiddlewareOutputSpider(Spider): - start_urls = ["http://example.com/"] +class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_output' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -205,8 +203,7 @@ class InvalidReturnValueOutputMiddleware(object): # ================================================================================ # make sure only non already called process_spider_output methods # are called if process_spider_exception returns an iterable -class ExecutionChainSpider(Spider): - start_urls = ["http://example.com"] +class ExecutionChainSpider(LocalhostSpider): name = 'execution_chain' custom_settings = { 'SPIDER_MIDDLEWARES': { @@ -258,6 +255,13 @@ class ThirdMiddleware(object): class TestSpiderMiddleware(TestCase): + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + @defer.inlineCallbacks def test_process_spider_exception_from_parse_method(self): # non-generator return value @@ -309,8 +313,8 @@ class TestSpiderMiddleware(TestCase): self.assertIn("ImportError exception caught", str(log)) self.assertIn("{'value': 10}", str(log)) self.assertIn("{'value': 'asdf'}", str(log)) - self.assertIn("{'value': 'http://example.com'}", str(log)) - self.assertIn("{'value': 'http://example.org'}", str(log)) + self.assertIn("{'value': 'http://localhost:8998'}", str(log)) + self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log)) @defer.inlineCallbacks def test_process_spider_exception_invalid_return_value_previous_middleware(self): From 9c256cf693d73e854d409d717854b3f354b5e0a9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Mar 2017 15:41:57 -0300 Subject: [PATCH 003/140] Undocument _InvalidOutput exception --- docs/topics/exceptions.rst | 11 ----------- scrapy/core/spidermw.py | 12 ++++++------ scrapy/exceptions.py | 8 +++++--- tests/test_spidermiddleware.py | 12 ++++++------ 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst index a3ff72827..cc02369d4 100644 --- a/docs/topics/exceptions.rst +++ b/docs/topics/exceptions.rst @@ -62,17 +62,6 @@ remain disabled. Those components include: The exception must be raised in the component's ``__init__`` method. -InvalidOutput -------------- - -.. exception:: InvalidOutput - -This exception can be raised by a downloader or spider middleware to -indicate that some method returned a value not suported by the processing -chain. -See :ref:`topics-spider-middleware` and :ref:`topics-downloader-middleware` -for a list of supported output values. - NotSupported ------------ diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 0f03a7b36..50677670b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -5,7 +5,7 @@ See documentation in docs/topics/spider-middleware.rst """ import six from twisted.python.failure import Failure -from scrapy.exceptions import InvalidOutput +from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list @@ -42,7 +42,7 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - raise InvalidOutput('Middleware {} must return None or raise ' \ + raise _InvalidOutput('Middleware {} must return None or raise ' \ 'an exception, got {}'.format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) @@ -50,13 +50,13 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_spider_exception(_failure): exception = _failure.value - # don't handle InvalidOutput exception - if isinstance(exception, InvalidOutput): + # don't handle _InvalidOutput exception + if isinstance(exception, _InvalidOutput): return _failure for method in self.methods['process_spider_exception']: result = method(response=response, exception=exception, spider=spider) if result is not None and not _isiterable(result): - raise InvalidOutput('Middleware {} must return None or an iterable ' \ + raise _InvalidOutput('Middleware {} must return None or an iterable ' \ 'object, got {}'.format(fname(method), type(result))) # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned @@ -80,7 +80,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if _isiterable(result): result = wrapper(result) else: - raise InvalidOutput('Middleware {} must return an iterable object, ' \ + raise _InvalidOutput('Middleware {} must return an iterable object, ' \ 'got {}'.format(fname(method), type(result))) return result diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index ba7272255..96949bdd9 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -11,9 +11,11 @@ class NotConfigured(Exception): """Indicates a missing configuration situation""" pass -class InvalidOutput(TypeError): - """Indicates an invalid value has been returned - by a middleware's processing method""" +class _InvalidOutput(TypeError): + """ + Indicates an invalid value has been returned by a middleware's processing method. + Internal and undocumented, it should not be raised or caught by user code. + """ pass # HTTP and crawling diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 3981a8d75..2d05c335c 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -159,7 +159,7 @@ class DoSomethingMiddleware(object): # ================================================================================ -# don't catch InvalidOutput from scrapy's spider middleware manager +# don't catch _InvalidOutput from scrapy's spider middleware manager class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): name = 'invalid_return_value_from_previous_middleware_input' custom_settings = { @@ -318,19 +318,19 @@ class TestSpiderMiddleware(TestCase): @defer.inlineCallbacks def test_process_spider_exception_invalid_return_value_previous_middleware(self): - """ don't catch InvalidOutput from middleware """ + """ don't catch _InvalidOutput from middleware """ # on middleware's input crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) with LogCapture() as log1: yield crawler1.crawl() - self.assertNotIn("InvalidOutput exception caught", str(log1)) - self.assertIn("'spider_exceptions/InvalidOutput'", str(log1)) + self.assertNotIn("_InvalidOutput exception caught", str(log1)) + self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) # on middleware's output crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) with LogCapture() as log2: yield crawler2.crawl() - self.assertNotIn("InvalidOutput exception caught", str(log2)) - self.assertIn("'spider_exceptions/InvalidOutput'", str(log2)) + self.assertNotIn("_InvalidOutput exception caught", str(log2)) + self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) @defer.inlineCallbacks def test_process_spider_exception_execution_chain(self): From 4cfbe8204480214b65d48caaf080feda30fe91ae Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 10 Mar 2017 15:47:47 -0300 Subject: [PATCH 004/140] Downloader middleware: raise _InvalidOutput Instead of AssertionError, to make it consistent with spider middleware --- scrapy/core/downloader/middleware.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index c3b23e284..cf0c1f869 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -7,6 +7,7 @@ import six from twisted.internet import defer +from scrapy.exceptions import _InvalidOutput from scrapy.http import Request, Response from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred @@ -35,9 +36,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_request(request): for method in self.methods['process_request']: response = yield method(request=request, spider=spider) - assert response is None or isinstance(response, (Response, Request)), \ - 'Middleware %s.process_request must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, response.__class__.__name__) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \ + (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) defer.returnValue((yield download_func(request=request,spider=spider))) @@ -51,9 +52,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_response']: response = yield method(request=request, response=response, spider=spider) - assert isinstance(response, (Response, Request)), \ - 'Middleware %s.process_response must return Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response)) + if not isinstance(response, (Response, Request)): + raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ + (six.get_method_self(method).__class__.__name__, type(response))) if isinstance(response, Request): defer.returnValue(response) defer.returnValue(response) @@ -64,9 +65,9 @@ class DownloaderMiddlewareManager(MiddlewareManager): for method in self.methods['process_exception']: response = yield method(request=request, exception=exception, spider=spider) - assert response is None or isinstance(response, (Response, Request)), \ - 'Middleware %s.process_exception must return None, Response or Request, got %s' % \ - (six.get_method_self(method).__class__.__name__, type(response)) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ + (six.get_method_self(method).__class__.__name__, type(response))) if response: defer.returnValue(response) defer.returnValue(_failure) From b040df5ac09cf133cd07b505d20469a56409129c Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 21 Mar 2017 15:56:18 +0500 Subject: [PATCH 005/140] TST cleanup spider middleware tests --- tests/test_spidermiddleware.py | 73 ++++++++++++++++------------------ 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2d05c335c..109bcc250 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -255,61 +255,62 @@ class ThirdMiddleware(object): class TestSpiderMiddleware(TestCase): - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() - def tearDown(self): - self.mockserver.__exit__(None, None, None) + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method(self): - # non-generator return value - crawler = get_crawler(NotAGeneratorSpider) + def crawl_log(self, spider): + crawler = get_crawler(spider) with LogCapture() as log: yield crawler.crawl() + raise defer.returnValue(log) + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method_non_generator(self): + # non-generator return value + log = yield self.crawl_log(NotAGeneratorSpider) self.assertIn("AssertionError exception caught", str(log)) self.assertIn("spider_exceptions/AssertionError", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method_generator_no_items(self): # generator return value, no items before the error - crawler = get_crawler(GeneratorErrorBeforeItemsSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider) self.assertIn("ValueError exception caught", str(log)) self.assertIn("spider_exceptions/ValueError", str(log)) + + @defer.inlineCallbacks + def test_process_spider_exception_from_parse_method_generator_with_items(self): # generator return value, 3 items before the error - crawler = get_crawler(GeneratorErrorAfterItemsSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(GeneratorErrorAfterItemsSpider) self.assertIn("'item_scraped_count': 3", str(log)) self.assertIn("FloatingPointError exception caught", str(log)) self.assertIn("spider_exceptions/FloatingPointError", str(log)) @defer.inlineCallbacks def test_process_spider_exception_from_previous_middleware_input(self): - crawler = get_crawler(FromPreviousMiddlewareInputSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(FromPreviousMiddlewareInputSpider) self.assertIn("LookupError exception caught", str(log)) @defer.inlineCallbacks def test_process_spider_exception_from_previous_middleware_output(self): # non-generator output value - crawler = get_crawler(NotAGeneratorFromPreviousMiddlewareOutputSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider) self.assertNotIn("UnicodeError exception caught", str(log)) # generator output value - crawler = get_crawler(GeneratorFromPreviousMiddlewareOutputSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider) self.assertIn("'item_scraped_count': 3", str(log)) self.assertIn("NameError exception caught", str(log)) @defer.inlineCallbacks def test_process_spider_exception_do_something(self): - crawler = get_crawler(DoSomethingSpider) - with LogCapture() as log: - yield crawler.crawl() + log = yield self.crawl_log(DoSomethingSpider) self.assertIn("ImportError exception caught", str(log)) self.assertIn("{'value': 10}", str(log)) self.assertIn("{'value': 'asdf'}", str(log)) @@ -320,25 +321,19 @@ class TestSpiderMiddleware(TestCase): def test_process_spider_exception_invalid_return_value_previous_middleware(self): """ don't catch _InvalidOutput from middleware """ # on middleware's input - crawler1 = get_crawler(InvalidReturnValueFromPreviousMiddlewareInputSpider) - with LogCapture() as log1: - yield crawler1.crawl() + log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider) self.assertNotIn("_InvalidOutput exception caught", str(log1)) self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) # on middleware's output - crawler2 = get_crawler(InvalidReturnValueFromPreviousMiddlewareOutputSpider) - with LogCapture() as log2: - yield crawler2.crawl() + log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider) self.assertNotIn("_InvalidOutput exception caught", str(log2)) self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) @defer.inlineCallbacks def test_process_spider_exception_execution_chain(self): # on middleware's input - crawler1 = get_crawler(ExecutionChainSpider) - with LogCapture() as log1: - yield crawler1.crawl() - self.assertNotIn("handled_by_first_middleware", str(log1)) - self.assertNotIn("handled_by_second_middleware", str(log1)) - self.assertIn("MemoryError exception caught", str(log1)) - self.assertIn("handled_by_third_middleware", str(log1)) + log = yield self.crawl_log(ExecutionChainSpider) + self.assertNotIn("handled_by_first_middleware", str(log)) + self.assertNotIn("handled_by_second_middleware", str(log)) + self.assertIn("MemoryError exception caught", str(log)) + self.assertIn("handled_by_third_middleware", str(log)) From 4740dca8f260bef83eed849b692b3a2c1aaec6cf Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 24 Jun 2018 20:59:18 -0300 Subject: [PATCH 006/140] Deferred-like process_output/process_exception chain --- scrapy/core/spidermw.py | 58 +++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 22 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 50677670b..98e264bd3 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -17,6 +17,11 @@ class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' + # List of dicts. Each dict represents a spider middleware and contains the + # 'process_spider_output', 'process_spider_exception' methods. + # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain + output_methods = [] + @classmethod def _get_mwlist_from_settings(cls, settings): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) @@ -25,12 +30,12 @@ class SpiderMiddlewareManager(MiddlewareManager): super(SpiderMiddlewareManager, self)._add_middleware(mw) if hasattr(mw, 'process_spider_input'): self.methods['process_spider_input'].append(mw.process_spider_input) - if hasattr(mw, 'process_spider_output'): - self.methods['process_spider_output'].insert(0, mw.process_spider_output) - if hasattr(mw, 'process_spider_exception'): - self.methods['process_spider_exception'].insert(0, mw.process_spider_exception) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].insert(0, mw.process_start_requests) + self.output_methods.insert(0, dict( + process_spider_output=getattr(mw, 'process_spider_output', None), + process_spider_exception=getattr(mw, 'process_spider_exception', None), + )) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( @@ -48,45 +53,54 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure): + def process_spider_exception(_failure, mw_index): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for method in self.methods['process_spider_exception']: - result = method(response=response, exception=exception, spider=spider) - if result is not None and not _isiterable(result): - raise _InvalidOutput('Middleware {} must return None or an iterable ' \ - 'object, got {}'.format(fname(method), type(result))) - # stop exception handling by handing control over to the - # process_spider_output chain if an iterable has been returned - if result is not None: - return result + for index, mw in enumerate(self.output_methods): + if index < mw_index or mw['process_spider_exception'] is None: + continue + result = mw['process_spider_exception'](response=response, exception=exception, spider=spider) + mw_index += 1 + if _isiterable(result): + # stop exception handling by handing control over to the + # process_spider_output chain if an iterable has been returned + return process_spider_output(result, mw_index) + elif result is None: + continue + else: + raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \ + .format(fname(mw['process_spider_exception']), type(result))) return _failure - def process_spider_output(result): + def process_spider_output(result, mw_index): def wrapper(result_iterable): try: for r in result_iterable: yield r except Exception as ex: - exception_result = process_spider_exception(Failure(ex)) + # process the exception with the method from the next middleware + exception_result = process_spider_exception(Failure(ex), mw_index) if exception_result is None or isinstance(exception_result, Failure): raise for output in exception_result: yield output - for method in self.methods['process_spider_output']: - result = method(response=response, result=result, spider=spider) + for index, mw in enumerate(self.output_methods): + if index < mw_index or mw['process_spider_output'] is None: + continue + result = mw['process_spider_output'](response=response, result=result, spider=spider) + mw_index += 1 if _isiterable(result): result = wrapper(result) else: - raise _InvalidOutput('Middleware {} must return an iterable object, ' \ - 'got {}'.format(fname(method), type(result))) + raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ + .format(fname(mw['process_spider_output']), type(result))) return result dfd = mustbe_deferred(process_spider_input, response) - dfd.addErrback(process_spider_exception) - dfd.addCallback(process_spider_output) + dfd.addErrback(process_spider_exception, mw_index=0) + dfd.addCallback(process_spider_output, mw_index=0) return dfd def process_start_requests(self, start_requests, spider): From ba294351381c0dd81476603246d2cea6c31486be Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 25 Jun 2018 15:01:12 -0300 Subject: [PATCH 007/140] Default values for OffsiteMiddleware For some reason test_crawl.py seems to be skipping the spider_opened method, which initializes the host_regex instance variable --- scrapy/spidermiddlewares/offsite.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 310166cad..3b7f194e4 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -19,6 +19,9 @@ class OffsiteMiddleware(object): def __init__(self, stats): self.stats = stats + # default values + self.host_regex = re.compile('') # allow all by default + self.domains_seen = set() @classmethod def from_crawler(cls, crawler): @@ -52,7 +55,7 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return re.compile('') # allow all by default + return url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): @@ -62,8 +65,9 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - self.host_regex = self.get_host_regex(spider) - self.domains_seen = set() + host_regex = self.get_host_regex(spider) + if host_regex: + self.host_regex = host_regex class URLWarning(Warning): From df75a0942e004f9645182a0260769f4337f843e5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 1 Jul 2018 13:30:50 -0300 Subject: [PATCH 008/140] Update docs --- docs/topics/spider-middleware.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0af26be73..dde1786af 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -112,9 +112,8 @@ following methods: .. method:: process_spider_exception(response, exception, spider) - This method is called when when a spider or :meth:`process_spider_input`/ - :meth:`process_spider_output` method (from other spider middleware) - raises an exception. + This method is called when a spider or :meth:`process_spider_output` + method (from a previous spider middleware) raises an exception. :meth:`process_spider_exception` should return either ``None`` or an iterable of :class:`~scrapy.http.Request`, dict or @@ -126,7 +125,7 @@ following methods: exception reaches the engine (where it's logged and discarded). If it returns an iterable the :meth:`process_spider_output` pipeline - kicks in, starting with the last non-executed method, and no other + kicks in, starting from the next spider middleware, and no other :meth:`process_spider_exception` will be called. :param response: the response being processed when the exception was From 735de8167d3e6b0085710d406c8c2976913baa43 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 30 Jun 2018 20:55:17 -0300 Subject: [PATCH 009/140] Test for exceptions on process_spider_input --- tests/test_spider_mw.py | 77 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_spider_mw.py diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py new file mode 100644 index 000000000..4a431d379 --- /dev/null +++ b/tests/test_spider_mw.py @@ -0,0 +1,77 @@ + +import logging + +from testfixtures import LogCapture +from twisted.trial.unittest import TestCase +from twisted.internet import defer + +from scrapy import Spider, Request +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + + +class CommonTestCase(TestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def crawl_log(self, spider): + crawler = get_crawler(spider) + with LogCapture() as log: + yield crawler.crawl() + raise defer.returnValue(log) + + +class LogExceptionMiddleware(object): + def process_spider_exception(self, response, exception, spider): + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + return None + + +# ================================================================================ +# (1) exceptions from a spider middleware's process_spider_input method +class ProcessSpiderInputSpider(Spider): + name = 'ProcessSpiderInputSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + __name__ + '.FailProcessSpiderInputMiddleware': 8, + __name__ + '.LogExceptionMiddleware': 6, + # engine + } + } + + def start_requests(self): + yield Request('http://localhost:8998', callback=self.parse, errback=self.errback) + + def parse(self, response): + return [{'test': 1}, {'test': 2}] + + def errback(self, failure): + self.logger.warn('Got a Failure on the Request errback') + + +class FailProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + logging.warn('Middleware: will raise ZeroDivisionError') + raise ZeroDivisionError() + + +class TestProcessSpiderInputSpider(CommonTestCase): + @defer.inlineCallbacks + def test_process_spider_input_errback(self): + """ + (1) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain, it should go directly to the Request errback + """ + log = yield self.crawl_log(ProcessSpiderInputSpider) + self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log)) + self.assertIn('Middleware: will raise ZeroDivisionError', str(log)) + self.assertIn('Got a Failure on the Request errback', str(log)) From 6ed9440ed528ab5c5eece50512e19929e4320b42 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 30 Jun 2018 21:27:10 -0300 Subject: [PATCH 010/140] Tests for exceptions on spider callbacks --- tests/test_spider_mw.py | 72 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py index 4a431d379..092546291 100644 --- a/tests/test_spider_mw.py +++ b/tests/test_spider_mw.py @@ -60,11 +60,11 @@ class ProcessSpiderInputSpider(Spider): class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): - logging.warn('Middleware: will raise ZeroDivisionError') - raise ZeroDivisionError() + logging.warn('Middleware: will raise IndexError') + raise IndexError() -class TestProcessSpiderInputSpider(CommonTestCase): +class TestProcessSpiderInput(CommonTestCase): @defer.inlineCallbacks def test_process_spider_input_errback(self): """ @@ -72,6 +72,66 @@ class TestProcessSpiderInputSpider(CommonTestCase): process_spider_exception chain, it should go directly to the Request errback """ log = yield self.crawl_log(ProcessSpiderInputSpider) - self.assertNotIn('Middleware: ZeroDivisionError exception caught', str(log)) - self.assertIn('Middleware: will raise ZeroDivisionError', str(log)) - self.assertIn('Got a Failure on the Request errback', str(log)) + self.assertNotIn("Middleware: IndexError exception caught", str(log)) + self.assertIn("Middleware: will raise IndexError", str(log)) + self.assertIn("Got a Failure on the Request errback", str(log)) + + +# ================================================================================ +# (2) exceptions from a spider callback (generator) +class GeneratorCallbackSpider(Spider): + name = 'GeneratorCallbackSpider' + start_urls = ['http://localhost:8998'] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + # engine + }, + } + + def parse(self, response): + yield {'test': 1} + yield {'test': 2} + raise ImportError() + + +class TestGeneratorCallback(CommonTestCase): + @defer.inlineCallbacks + def test_generator_callback(self): + """ + (2) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log = yield self.crawl_log(GeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log)) + self.assertIn("'item_scraped_count': 2", str(log)) + + +# ================================================================================ +# (3) exceptions from a spider callback (not a generator) +class NotAGeneratorCallbackSpider(Spider): + name = 'NotAGeneratorCallbackSpider' + start_urls = ['http://localhost:8998'] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + # engine + }, + } + + def parse(self, response): + return [{'test': 1}, {'test': 1/0}] + + +class TestNotAGeneratorCallback(CommonTestCase): + @defer.inlineCallbacks + def test_not_a_generator_callback(self): + """ + (3) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log = yield self.crawl_log(NotAGeneratorCallbackSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log)) + self.assertNotIn("item_scraped_count", str(log)) From 4fca9aba851133fcdc12bb46c7ae229d9537079a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 1 Jul 2018 13:18:29 -0300 Subject: [PATCH 011/140] Recover from a callback exception --- tests/test_spider_mw.py | 137 +++++++++++++++++++++++++--------------- 1 file changed, 85 insertions(+), 52 deletions(-) diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py index 092546291..49a60d08b 100644 --- a/tests/test_spider_mw.py +++ b/tests/test_spider_mw.py @@ -10,22 +10,8 @@ from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class CommonTestCase(TestCase): - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def crawl_log(self, spider): - crawler = get_crawler(spider) - with LogCapture() as log: - yield crawler.crawl() - raise defer.returnValue(log) +# TEST_URL = 'http://example.org' +TEST_URL = 'http://localhost:8998' class LogExceptionMiddleware(object): @@ -34,6 +20,32 @@ class LogExceptionMiddleware(object): return None +# ================================================================================ +# recover from an exception on a spider callback +class RecoverySpider(Spider): + name = 'RecoverySpider' + start_urls = [TEST_URL] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.RecoveryMiddleware': 10, + }, + } + + def parse(self, response): + yield {'test': 1} + self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) + if not response.meta.get('dont_fail'): + raise ModuleNotFoundError() + +class RecoveryMiddleware(object): + def process_spider_exception(self, response, exception, spider): + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] + + # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method class ProcessSpiderInputSpider(Spider): @@ -49,7 +61,7 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request('http://localhost:8998', callback=self.parse, errback=self.errback) + yield Request(TEST_URL, callback=self.parse, errback=self.errback) def parse(self, response): return [{'test': 1}, {'test': 2}] @@ -64,29 +76,14 @@ class FailProcessSpiderInputMiddleware: raise IndexError() -class TestProcessSpiderInput(CommonTestCase): - @defer.inlineCallbacks - def test_process_spider_input_errback(self): - """ - (1) An exception from the process_spider_input chain should not be caught by the - process_spider_exception chain, it should go directly to the Request errback - """ - log = yield self.crawl_log(ProcessSpiderInputSpider) - self.assertNotIn("Middleware: IndexError exception caught", str(log)) - self.assertIn("Middleware: will raise IndexError", str(log)) - self.assertIn("Got a Failure on the Request errback", str(log)) - - # ================================================================================ # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' - start_urls = ['http://localhost:8998'] + start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { - # spider __name__ + '.LogExceptionMiddleware': 10, - # engine }, } @@ -96,28 +93,14 @@ class GeneratorCallbackSpider(Spider): raise ImportError() -class TestGeneratorCallback(CommonTestCase): - @defer.inlineCallbacks - def test_generator_callback(self): - """ - (2) An exception from a spider's callback should - be caught by the process_spider_exception chain - """ - log = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log)) - self.assertIn("'item_scraped_count': 2", str(log)) - - # ================================================================================ # (3) exceptions from a spider callback (not a generator) class NotAGeneratorCallbackSpider(Spider): name = 'NotAGeneratorCallbackSpider' - start_urls = ['http://localhost:8998'] + start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { - # spider __name__ + '.LogExceptionMiddleware': 10, - # engine }, } @@ -125,13 +108,63 @@ class NotAGeneratorCallbackSpider(Spider): return [{'test': 1}, {'test': 1/0}] -class TestNotAGeneratorCallback(CommonTestCase): +# ================================================================================ +class TestSpiderMiddleware(TestCase): + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def crawl_log(self, spider): + crawler = get_crawler(spider) + with LogCapture() as log: + yield crawler.crawl() + raise defer.returnValue(log) + + # @defer.inlineCallbacks + # def test_recovery(self): + # """ + # Recover from an exception from a spider's callback. The final item count should be 3 + # (one from the spider before raising the exception, one from the middleware and one + # from the spider when processing the response that was enqueued from the middleware) + # """ + # log = yield self.crawl_log(RecoverySpider) + # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + # self.assertIn("'item_scraped_count': 3", str(log)) + + @defer.inlineCallbacks + def test_process_spider_input_errback(self): + """ + (1) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain, it should go directly to the Request errback + """ + log1 = yield self.crawl_log(ProcessSpiderInputSpider) + self.assertNotIn("Middleware: IndexError exception caught", str(log1)) + self.assertIn("Middleware: will raise IndexError", str(log1)) + self.assertIn("Got a Failure on the Request errback", str(log1)) + + @defer.inlineCallbacks + def test_generator_callback(self): + """ + (2) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log2 = yield self.crawl_log(GeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks def test_not_a_generator_callback(self): """ (3) An exception from a spider's callback should be caught by the process_spider_exception chain """ - log = yield self.crawl_log(NotAGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log)) - self.assertNotIn("item_scraped_count", str(log)) + log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) + self.assertNotIn("item_scraped_count", str(log3)) From 985ab636cfa0825f100b02583bcfd106d1f4cef6 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 1 Jul 2018 17:49:30 -0300 Subject: [PATCH 012/140] Store output methods on the 'methods' dict --- scrapy/core/spidermw.py | 43 +++++++++++++++++------------------------ 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 98e264bd3..c9dd8c91e 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -17,11 +17,6 @@ class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' - # List of dicts. Each dict represents a spider middleware and contains the - # 'process_spider_output', 'process_spider_exception' methods. - # The idea is to simulate the behaviour of a Twisted deferred's callback/errback chain - output_methods = [] - @classmethod def _get_mwlist_from_settings(cls, settings): return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES')) @@ -32,10 +27,8 @@ class SpiderMiddlewareManager(MiddlewareManager): self.methods['process_spider_input'].append(mw.process_spider_input) if hasattr(mw, 'process_start_requests'): self.methods['process_start_requests'].insert(0, mw.process_start_requests) - self.output_methods.insert(0, dict( - process_spider_output=getattr(mw, 'process_spider_output', None), - process_spider_exception=getattr(mw, 'process_spider_exception', None), - )) + self.methods['process_spider_output'].insert(0, getattr(mw, 'process_spider_output', None)) + self.methods['process_spider_exception'].insert(0, getattr(mw, 'process_spider_exception', None)) def scrape_response(self, scrape_func, response, request, spider): fname = lambda f:'%s.%s' % ( @@ -53,54 +46,54 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure, mw_index): + def process_spider_exception(_failure, index): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for index, mw in enumerate(self.output_methods): - if index < mw_index or mw['process_spider_exception'] is None: + for i, method in enumerate(self.methods['process_spider_exception']): + if i < index or method is None: continue - result = mw['process_spider_exception'](response=response, exception=exception, spider=spider) - mw_index += 1 + result = method(response=response, exception=exception, spider=spider) + index += 1 if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return process_spider_output(result, mw_index) + return process_spider_output(result, index) elif result is None: continue else: raise _InvalidOutput('Middleware {} must return None or an iterable, got {}' \ - .format(fname(mw['process_spider_exception']), type(result))) + .format(fname(method), type(result))) return _failure - def process_spider_output(result, mw_index): + def process_spider_output(result, index): def wrapper(result_iterable): try: for r in result_iterable: yield r except Exception as ex: # process the exception with the method from the next middleware - exception_result = process_spider_exception(Failure(ex), mw_index) + exception_result = process_spider_exception(Failure(ex), index) if exception_result is None or isinstance(exception_result, Failure): raise for output in exception_result: yield output - for index, mw in enumerate(self.output_methods): - if index < mw_index or mw['process_spider_output'] is None: + for i, method in enumerate(self.methods['process_spider_output']): + if i < index or method is None: continue - result = mw['process_spider_output'](response=response, result=result, spider=spider) - mw_index += 1 + result = method(response=response, result=result, spider=spider) + index += 1 if _isiterable(result): result = wrapper(result) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ - .format(fname(mw['process_spider_output']), type(result))) + .format(fname(method), type(result))) return result dfd = mustbe_deferred(process_spider_input, response) - dfd.addErrback(process_spider_exception, mw_index=0) - dfd.addCallback(process_spider_output, mw_index=0) + dfd.addErrback(process_spider_exception, index=0) + dfd.addCallback(process_spider_output, index=0) return dfd def process_start_requests(self, start_requests, spider): From 0b2870634af6cc14191faafcba7d58a5f3cc3016 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 4 Jul 2018 16:14:51 -0300 Subject: [PATCH 013/140] Do not inherit from object --- tests/test_spider_mw.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py index 49a60d08b..2565ef7af 100644 --- a/tests/test_spider_mw.py +++ b/tests/test_spider_mw.py @@ -14,7 +14,7 @@ from tests.mockserver import MockServer TEST_URL = 'http://localhost:8998' -class LogExceptionMiddleware(object): +class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): logging.warn('Middleware: %s exception caught', exception.__class__.__name__) return None @@ -37,7 +37,7 @@ class RecoverySpider(Spider): if not response.meta.get('dont_fail'): raise ModuleNotFoundError() -class RecoveryMiddleware(object): +class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): logging.warn('Middleware: %s exception caught', exception.__class__.__name__) return [ From 0a0e62272e67aaebe29666017e9b0623b81bf369 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 4 Jul 2018 16:19:19 -0300 Subject: [PATCH 014/140] New tests --- tests/test_spider_mw.py | 170 -------------- tests/test_spidermiddleware.py | 405 ++++++++++----------------------- 2 files changed, 118 insertions(+), 457 deletions(-) delete mode 100644 tests/test_spider_mw.py diff --git a/tests/test_spider_mw.py b/tests/test_spider_mw.py deleted file mode 100644 index 2565ef7af..000000000 --- a/tests/test_spider_mw.py +++ /dev/null @@ -1,170 +0,0 @@ - -import logging - -from testfixtures import LogCapture -from twisted.trial.unittest import TestCase -from twisted.internet import defer - -from scrapy import Spider, Request -from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer - - -# TEST_URL = 'http://example.org' -TEST_URL = 'http://localhost:8998' - - -class LogExceptionMiddleware: - def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) - return None - - -# ================================================================================ -# recover from an exception on a spider callback -class RecoverySpider(Spider): - name = 'RecoverySpider' - start_urls = [TEST_URL] - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.RecoveryMiddleware': 10, - }, - } - - def parse(self, response): - yield {'test': 1} - self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) - if not response.meta.get('dont_fail'): - raise ModuleNotFoundError() - -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) - return [ - {'from': 'process_spider_exception'}, - Request(response.url, meta={'dont_fail': True}, dont_filter=True), - ] - - -# ================================================================================ -# (1) exceptions from a spider middleware's process_spider_input method -class ProcessSpiderInputSpider(Spider): - name = 'ProcessSpiderInputSpider' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # spider - __name__ + '.LogExceptionMiddleware': 10, - __name__ + '.FailProcessSpiderInputMiddleware': 8, - __name__ + '.LogExceptionMiddleware': 6, - # engine - } - } - - def start_requests(self): - yield Request(TEST_URL, callback=self.parse, errback=self.errback) - - def parse(self, response): - return [{'test': 1}, {'test': 2}] - - def errback(self, failure): - self.logger.warn('Got a Failure on the Request errback') - - -class FailProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): - logging.warn('Middleware: will raise IndexError') - raise IndexError() - - -# ================================================================================ -# (2) exceptions from a spider callback (generator) -class GeneratorCallbackSpider(Spider): - name = 'GeneratorCallbackSpider' - start_urls = [TEST_URL] - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, - }, - } - - def parse(self, response): - yield {'test': 1} - yield {'test': 2} - raise ImportError() - - -# ================================================================================ -# (3) exceptions from a spider callback (not a generator) -class NotAGeneratorCallbackSpider(Spider): - name = 'NotAGeneratorCallbackSpider' - start_urls = [TEST_URL] - custom_settings = { - 'SPIDER_MIDDLEWARES': { - __name__ + '.LogExceptionMiddleware': 10, - }, - } - - def parse(self, response): - return [{'test': 1}, {'test': 1/0}] - - -# ================================================================================ -class TestSpiderMiddleware(TestCase): - @classmethod - def setUpClass(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def tearDownClass(cls): - cls.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def crawl_log(self, spider): - crawler = get_crawler(spider) - with LogCapture() as log: - yield crawler.crawl() - raise defer.returnValue(log) - - # @defer.inlineCallbacks - # def test_recovery(self): - # """ - # Recover from an exception from a spider's callback. The final item count should be 3 - # (one from the spider before raising the exception, one from the middleware and one - # from the spider when processing the response that was enqueued from the middleware) - # """ - # log = yield self.crawl_log(RecoverySpider) - # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - # self.assertIn("'item_scraped_count': 3", str(log)) - - @defer.inlineCallbacks - def test_process_spider_input_errback(self): - """ - (1) An exception from the process_spider_input chain should not be caught by the - process_spider_exception chain, it should go directly to the Request errback - """ - log1 = yield self.crawl_log(ProcessSpiderInputSpider) - self.assertNotIn("Middleware: IndexError exception caught", str(log1)) - self.assertIn("Middleware: will raise IndexError", str(log1)) - self.assertIn("Got a Failure on the Request errback", str(log1)) - - @defer.inlineCallbacks - def test_generator_callback(self): - """ - (2) An exception from a spider's callback should - be caught by the process_spider_exception chain - """ - log2 = yield self.crawl_log(GeneratorCallbackSpider) - self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) - - @defer.inlineCallbacks - def test_not_a_generator_callback(self): - """ - (3) An exception from a spider's callback should - be caught by the process_spider_exception chain - """ - log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) - self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) - self.assertNotIn("item_scraped_count", str(log3)) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 109bcc250..2565ef7af 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -5,256 +5,111 @@ from testfixtures import LogCapture from twisted.trial.unittest import TestCase from twisted.internet import defer -from scrapy.spiders import Spider -from scrapy.item import Item, Field -from scrapy.http import Request +from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer -class TestItem(Item): - value = Field() +# TEST_URL = 'http://example.org' +TEST_URL = 'http://localhost:8998' -class LocalhostSpider(Spider): - start_urls = ['http://localhost:8998'] # tests.mockserver.MockServer - - -# ================================================================================ -# exceptions from a spider's parse method -class BaseExceptionFromParseMethodSpider(LocalhostSpider): - custom_settings = { - 'SPIDER_MIDDLEWARES': {'tests.test_spidermiddleware.CatchExceptionMiddleware': 540} - } - - -class NotAGeneratorSpider(BaseExceptionFromParseMethodSpider): - """ return value is NOT a generator """ - name = 'not_a_generator' - - def parse(self, response): - raise AssertionError - - -class GeneratorErrorBeforeItemsSpider(BaseExceptionFromParseMethodSpider): - """ return value is a generator; the exception is raised - before the items are yielded: no items should be scraped """ - name = 'generator_error_before_items' - - def parse(self, response): - raise ValueError - for i in range(3): - yield {'value': i} - - -class GeneratorErrorAfterItemsSpider(BaseExceptionFromParseMethodSpider): - """ return value is a generator; the exception is raised - after the items are yielded: 3 items should be scraped """ - name = 'generator_error_after_items' - - def parse(self, response): - for i in range(3): - yield {'value': i} - raise FloatingPointError - - -class CatchExceptionMiddleware(object): +class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): - """ catch an exception and log it """ - logging.warn('{} exception caught'.format(exception.__class__.__name__)) + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) return None # ================================================================================ -# exception from a previous middleware's process_spider_input method -# process_spider_input is not expected to return an iterable, so there are no -# separate tests for generator/non-generator implementations -class FromPreviousMiddlewareInputSpider(LocalhostSpider): - name = 'not_a_generator_from_previous_middleware_input' +# recover from an exception on a spider callback +class RecoverySpider(Spider): + name = 'RecoverySpider' + start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.RaiseExceptionOnInputMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return None - - -class RaiseExceptionOnInputMiddleware(object): - def process_spider_input(self, response, spider): - raise LookupError - - -# ================================================================================ -# exception from a previous middleware's process_spider_output method (not a generator) -class NotAGeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): - name = 'not_a_generator_from_previous_middleware_output' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.RaiseExceptionOnOutputNotAGeneratorMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return [{'value': i} for i in range(3)] - - -class RaiseExceptionOnOutputNotAGeneratorMiddleware(object): - def process_spider_output(self, response, result, spider): - raise UnicodeError - - -# ================================================================================ -# exception from a previous middleware's process_spider_output method (generator) -class GeneratorFromPreviousMiddlewareOutputSpider(LocalhostSpider): - name = 'generator_from_previous_middleware_output' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.RaiseExceptionOnOutputGeneratorMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return [{'value': i} for i in range(10, 13)] - - -class RaiseExceptionOnOutputGeneratorMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - yield r - raise NameError - - -# ================================================================================ -# do something useful from the exception handler -class DoSomethingSpider(LocalhostSpider): - name = 'do_something' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.DoSomethingMiddleware': 540, - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, - # spider side - } - } - - def parse(self, response): - yield {'value': response.url} - raise ImportError - - -class DoSomethingMiddleware(object): - def process_spider_exception(self, response, exception, spider): - return [Request('http://localhost:8998?processed=true'), {'value': 10}, TestItem(value='asdf')] - - -# ================================================================================ -# don't catch _InvalidOutput from scrapy's spider middleware manager -class InvalidReturnValueFromPreviousMiddlewareInputSpider(LocalhostSpider): - name = 'invalid_return_value_from_previous_middleware_input' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.InvalidReturnValueInputMiddleware': 540, - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return None - - -class InvalidReturnValueInputMiddleware(object): - def process_spider_input(self, response, spider): - return 1.0 # , not None - - -class InvalidReturnValueFromPreviousMiddlewareOutputSpider(LocalhostSpider): - name = 'invalid_return_value_from_previous_middleware_output' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.CatchExceptionMiddleware': 540, - 'tests.test_spidermiddleware.InvalidReturnValueOutputMiddleware': 545, - # spider side - } - } - - def parse(self, response): - return None - - -class InvalidReturnValueOutputMiddleware(object): - def process_spider_output(self, response, result, spider): - return 1 # , not an iterable - - -# ================================================================================ -# make sure only non already called process_spider_output methods -# are called if process_spider_exception returns an iterable -class ExecutionChainSpider(LocalhostSpider): - name = 'execution_chain' - custom_settings = { - 'SPIDER_MIDDLEWARES': { - # engine side - 'tests.test_spidermiddleware.ThirdMiddleware': 540, - 'tests.test_spidermiddleware.SecondMiddleware': 541, - 'tests.test_spidermiddleware.FirstMiddleware': 542 - # spider side + __name__ + '.RecoveryMiddleware': 10, }, } def parse(self, response): - return None - - -class FirstMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - if isinstance(r, dict): - r['handled_by_first_middleware'] = True - yield r + yield {'test': 1} + self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) + if not response.meta.get('dont_fail'): + raise ModuleNotFoundError() +class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): - # log exception, handle control to the next middleware's process_spider_exception - logging.warn('{} exception caught'.format(exception.__class__.__name__)) - return None + logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + return [ + {'from': 'process_spider_exception'}, + Request(response.url, meta={'dont_fail': True}, dont_filter=True), + ] -class SecondMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - if isinstance(r, dict): - r['handled_by_second_middleware'] = True - yield r - raise MemoryError +# ================================================================================ +# (1) exceptions from a spider middleware's process_spider_input method +class ProcessSpiderInputSpider(Spider): + name = 'ProcessSpiderInputSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + # spider + __name__ + '.LogExceptionMiddleware': 10, + __name__ + '.FailProcessSpiderInputMiddleware': 8, + __name__ + '.LogExceptionMiddleware': 6, + # engine + } + } + + def start_requests(self): + yield Request(TEST_URL, callback=self.parse, errback=self.errback) + + def parse(self, response): + return [{'test': 1}, {'test': 2}] + + def errback(self, failure): + self.logger.warn('Got a Failure on the Request errback') -class ThirdMiddleware(object): - def process_spider_output(self, response, result, spider): - for r in result: - if isinstance(r, dict): - r['handled_by_third_middleware'] = True - yield r - - def process_spider_exception(self, response, exception, spider): - # handle control to the next middleware's process_spider_output - return [{'item': i} for i in range(3)] +class FailProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + logging.warn('Middleware: will raise IndexError') + raise IndexError() +# ================================================================================ +# (2) exceptions from a spider callback (generator) +class GeneratorCallbackSpider(Spider): + name = 'GeneratorCallbackSpider' + start_urls = [TEST_URL] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.LogExceptionMiddleware': 10, + }, + } + + def parse(self, response): + yield {'test': 1} + yield {'test': 2} + raise ImportError() + + +# ================================================================================ +# (3) exceptions from a spider callback (not a generator) +class NotAGeneratorCallbackSpider(Spider): + name = 'NotAGeneratorCallbackSpider' + start_urls = [TEST_URL] + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.LogExceptionMiddleware': 10, + }, + } + + def parse(self, response): + return [{'test': 1}, {'test': 1/0}] + + +# ================================================================================ class TestSpiderMiddleware(TestCase): - @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -263,7 +118,7 @@ class TestSpiderMiddleware(TestCase): @classmethod def tearDownClass(cls): cls.mockserver.__exit__(None, None, None) - + @defer.inlineCallbacks def crawl_log(self, spider): crawler = get_crawler(spider) @@ -271,69 +126,45 @@ class TestSpiderMiddleware(TestCase): yield crawler.crawl() raise defer.returnValue(log) - @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method_non_generator(self): - # non-generator return value - log = yield self.crawl_log(NotAGeneratorSpider) - self.assertIn("AssertionError exception caught", str(log)) - self.assertIn("spider_exceptions/AssertionError", str(log)) + # @defer.inlineCallbacks + # def test_recovery(self): + # """ + # Recover from an exception from a spider's callback. The final item count should be 3 + # (one from the spider before raising the exception, one from the middleware and one + # from the spider when processing the response that was enqueued from the middleware) + # """ + # log = yield self.crawl_log(RecoverySpider) + # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + # self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method_generator_no_items(self): - # generator return value, no items before the error - log = yield self.crawl_log(GeneratorErrorBeforeItemsSpider) - self.assertIn("ValueError exception caught", str(log)) - self.assertIn("spider_exceptions/ValueError", str(log)) - + def test_process_spider_input_errback(self): + """ + (1) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain, it should go directly to the Request errback + """ + log1 = yield self.crawl_log(ProcessSpiderInputSpider) + self.assertNotIn("Middleware: IndexError exception caught", str(log1)) + self.assertIn("Middleware: will raise IndexError", str(log1)) + self.assertIn("Got a Failure on the Request errback", str(log1)) + @defer.inlineCallbacks - def test_process_spider_exception_from_parse_method_generator_with_items(self): - # generator return value, 3 items before the error - log = yield self.crawl_log(GeneratorErrorAfterItemsSpider) - self.assertIn("'item_scraped_count': 3", str(log)) - self.assertIn("FloatingPointError exception caught", str(log)) - self.assertIn("spider_exceptions/FloatingPointError", str(log)) - + def test_generator_callback(self): + """ + (2) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log2 = yield self.crawl_log(GeneratorCallbackSpider) + self.assertIn("Middleware: ImportError exception caught", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) + @defer.inlineCallbacks - def test_process_spider_exception_from_previous_middleware_input(self): - log = yield self.crawl_log(FromPreviousMiddlewareInputSpider) - self.assertIn("LookupError exception caught", str(log)) - - @defer.inlineCallbacks - def test_process_spider_exception_from_previous_middleware_output(self): - # non-generator output value - log = yield self.crawl_log(NotAGeneratorFromPreviousMiddlewareOutputSpider) - self.assertNotIn("UnicodeError exception caught", str(log)) - # generator output value - log = yield self.crawl_log(GeneratorFromPreviousMiddlewareOutputSpider) - self.assertIn("'item_scraped_count': 3", str(log)) - self.assertIn("NameError exception caught", str(log)) - - @defer.inlineCallbacks - def test_process_spider_exception_do_something(self): - log = yield self.crawl_log(DoSomethingSpider) - self.assertIn("ImportError exception caught", str(log)) - self.assertIn("{'value': 10}", str(log)) - self.assertIn("{'value': 'asdf'}", str(log)) - self.assertIn("{'value': 'http://localhost:8998'}", str(log)) - self.assertIn("{'value': 'http://localhost:8998?processed=true'}", str(log)) - - @defer.inlineCallbacks - def test_process_spider_exception_invalid_return_value_previous_middleware(self): - """ don't catch _InvalidOutput from middleware """ - # on middleware's input - log1 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareInputSpider) - self.assertNotIn("_InvalidOutput exception caught", str(log1)) - self.assertIn("'spider_exceptions/_InvalidOutput'", str(log1)) - # on middleware's output - log2 = yield self.crawl_log(InvalidReturnValueFromPreviousMiddlewareOutputSpider) - self.assertNotIn("_InvalidOutput exception caught", str(log2)) - self.assertIn("'spider_exceptions/_InvalidOutput'", str(log2)) - - @defer.inlineCallbacks - def test_process_spider_exception_execution_chain(self): - # on middleware's input - log = yield self.crawl_log(ExecutionChainSpider) - self.assertNotIn("handled_by_first_middleware", str(log)) - self.assertNotIn("handled_by_second_middleware", str(log)) - self.assertIn("MemoryError exception caught", str(log)) - self.assertIn("handled_by_third_middleware", str(log)) + def test_not_a_generator_callback(self): + """ + (3) An exception from a spider's callback should + be caught by the process_spider_exception chain + """ + log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) + self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) + self.assertNotIn("item_scraped_count", str(log3)) From e7e18db179f2e45aa38a5bbdb0abba7d983cdce7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Jul 2018 14:04:35 -0300 Subject: [PATCH 015/140] Fix tests --- tests/test_spidermiddleware.py | 57 +++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2565ef7af..5622c3179 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -8,10 +8,7 @@ from twisted.internet import defer from scrapy import Spider, Request from scrapy.utils.test import get_crawler from tests.mockserver import MockServer - - -# TEST_URL = 'http://example.org' -TEST_URL = 'http://localhost:8998' +from tests.spiders import MockServerSpider class LogExceptionMiddleware: @@ -21,16 +18,18 @@ class LogExceptionMiddleware: # ================================================================================ -# recover from an exception on a spider callback +# (0) recover from an exception on a spider callback class RecoverySpider(Spider): name = 'RecoverySpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.RecoveryMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): yield {'test': 1} self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) @@ -61,10 +60,11 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request(TEST_URL, callback=self.parse, errback=self.errback) + yield Request(url=self.mockserver.url('/status?n=200'), + callback=self.parse, errback=self.errback) def parse(self, response): - return [{'test': 1}, {'test': 2}] + return {'from': 'callback'} def errback(self, failure): self.logger.warn('Got a Failure on the Request errback') @@ -80,13 +80,15 @@ class FailProcessSpiderInputMiddleware: # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): name = 'GeneratorCallbackSpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): yield {'test': 1} yield {'test': 2} @@ -97,13 +99,15 @@ class GeneratorCallbackSpider(Spider): # (3) exceptions from a spider callback (not a generator) class NotAGeneratorCallbackSpider(Spider): name = 'NotAGeneratorCallbackSpider' - start_urls = [TEST_URL] custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, }, } + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + def parse(self, response): return [{'test': 1}, {'test': 1/0}] @@ -123,20 +127,20 @@ class TestSpiderMiddleware(TestCase): def crawl_log(self, spider): crawler = get_crawler(spider) with LogCapture() as log: - yield crawler.crawl() + yield crawler.crawl(mockserver=self.mockserver) raise defer.returnValue(log) - # @defer.inlineCallbacks - # def test_recovery(self): - # """ - # Recover from an exception from a spider's callback. The final item count should be 3 - # (one from the spider before raising the exception, one from the middleware and one - # from the spider when processing the response that was enqueued from the middleware) - # """ - # log = yield self.crawl_log(RecoverySpider) - # self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - # self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - # self.assertIn("'item_scraped_count': 3", str(log)) + @defer.inlineCallbacks + def test_recovery(self): + """ + (0) Recover from an exception in a spider callback. The final item count should be 2 + (one directly from the recovery middleware and one from the spider when processing + the request that was enqueued from the recovery middleware) + """ + log = yield self.crawl_log(RecoverySpider) + self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + self.assertIn("'item_scraped_count': 2", str(log)) @defer.inlineCallbacks def test_process_spider_input_errback(self): @@ -148,21 +152,24 @@ class TestSpiderMiddleware(TestCase): self.assertNotIn("Middleware: IndexError exception caught", str(log1)) self.assertIn("Middleware: will raise IndexError", str(log1)) self.assertIn("Got a Failure on the Request errback", str(log1)) + self.assertIn("{'from': 'errback'}", str(log1)) + self.assertNotIn("{'from': 'callback'}", str(log1)) + self.assertIn("'item_scraped_count': 1", str(log1)) @defer.inlineCallbacks def test_generator_callback(self): """ - (2) An exception from a spider's callback should + (2) An exception from a spider callback (returning a generator) should be caught by the process_spider_exception chain """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertIn("'item_scraped_count': 2", str(log2)) + self.assertNotIn("item_scraped_count", str(log2)) @defer.inlineCallbacks def test_not_a_generator_callback(self): """ - (3) An exception from a spider's callback should + (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain """ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) From 0c579b5276f502832f375be316094b4244cf87c5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 14 Jul 2018 19:58:42 -0300 Subject: [PATCH 016/140] Untested experiment --- scrapy/core/spidermw.py | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9dd8c91e..a9aeb6dcc 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from itertools import chain + import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput @@ -10,9 +12,28 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list + def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') + +class MutableChain: + def __init__(self, *args): + self.data = chain(*args) + + def extend(self, iterable): + self.data = chain(self.data, iterable) + + def __iter__(self): + return self.data.__iter__() + + def __next__(self): # py3 + return self.data.__next__() + + def next(self): # py2 + return self.data.next() + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -68,28 +89,32 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def process_spider_output(result, index): - def wrapper(result_iterable): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + def evaluate_result(result_iterable, index): try: for r in result_iterable: yield r except Exception as ex: - # process the exception with the method from the next middleware exception_result = process_spider_exception(Failure(ex), index) if exception_result is None or isinstance(exception_result, Failure): raise - for output in exception_result: - yield output + recovered.extend(exception_result) + for i, method in enumerate(self.methods['process_spider_output']): if i < index or method is None: continue result = method(response=response, result=result, spider=spider) index += 1 if _isiterable(result): - result = wrapper(result) + result = evaluate_result(result, index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) - return result + + return chain(result, recovered) dfd = mustbe_deferred(process_spider_input, response) dfd.addErrback(process_spider_exception, index=0) From c5fa0ae6bc536d6bc5370c6f3b634c33848971e5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 14 Jul 2018 19:58:42 -0300 Subject: [PATCH 017/140] Untested experiment --- scrapy/core/spidermw.py | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c9dd8c91e..8ee42c2cf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,6 +3,8 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ +from itertools import chain + import six from twisted.python.failure import Failure from scrapy.exceptions import _InvalidOutput @@ -10,9 +12,31 @@ from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list + def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') + +class MutableChain: + """ + Thin wrapper around itertools.chain, allowing to add iterables "in-place" + """ + def __init__(self, *args): + self.data = chain(*args) + + def extend(self, *iterables): + self.data = chain(self.data, *iterables) + + def __iter__(self): + return self.data.__iter__() + + def __next__(self): # py3 + return self.data.__next__() + + def next(self): # py2 + return self.data.next() + + class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' @@ -68,28 +92,32 @@ class SpiderMiddlewareManager(MiddlewareManager): return _failure def process_spider_output(result, index): - def wrapper(result_iterable): + # items in this iterable do not need to go through the process_spider_output + # chain, they went through it already from the process_spider_exception method + recovered = MutableChain() + + def evaluate_result(result_iterable, index): try: for r in result_iterable: yield r except Exception as ex: - # process the exception with the method from the next middleware exception_result = process_spider_exception(Failure(ex), index) if exception_result is None or isinstance(exception_result, Failure): raise - for output in exception_result: - yield output + recovered.extend(exception_result) + for i, method in enumerate(self.methods['process_spider_output']): if i < index or method is None: continue result = method(response=response, result=result, spider=spider) index += 1 if _isiterable(result): - result = wrapper(result) + result = evaluate_result(result, index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) - return result + + return chain(result, recovered) dfd = mustbe_deferred(process_spider_input, response) dfd.addErrback(process_spider_exception, index=0) From cff9e8762512033da181293bab379b485aeffa66 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 15 Jul 2018 16:21:08 -0300 Subject: [PATCH 018/140] Fix tests --- tests/test_spidermiddleware.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 5622c3179..c33eb28ca 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -68,6 +68,7 @@ class ProcessSpiderInputSpider(Spider): def errback(self, failure): self.logger.warn('Got a Failure on the Request errback') + return {'from': 'errback'} class FailProcessSpiderInputMiddleware: @@ -133,14 +134,15 @@ class TestSpiderMiddleware(TestCase): @defer.inlineCallbacks def test_recovery(self): """ - (0) Recover from an exception in a spider callback. The final item count should be 2 - (one directly from the recovery middleware and one from the spider when processing - the request that was enqueued from the recovery middleware) + (0) Recover from an exception in a spider callback. The final item count should be 3 + (one yielded from the callback method before the exception is raised, one directly + from the recovery middleware and one from the spider when processing the request that + was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) - self.assertIn("'item_scraped_count': 2", str(log)) + self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks def test_process_spider_input_errback(self): @@ -164,7 +166,7 @@ class TestSpiderMiddleware(TestCase): """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) - self.assertNotIn("item_scraped_count", str(log2)) + self.assertIn("'item_scraped_count': 2", str(log2)) @defer.inlineCallbacks def test_not_a_generator_callback(self): From 60c2ef86f0c40d17219d3e3320072fb5b1ded412 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 15 Jul 2018 16:47:55 -0300 Subject: [PATCH 019/140] Revert "Default values for OffsiteMiddleware" This reverts commit ba294351381c0dd81476603246d2cea6c31486be. --- scrapy/spidermiddlewares/offsite.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 3b7f194e4..310166cad 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -19,9 +19,6 @@ class OffsiteMiddleware(object): def __init__(self, stats): self.stats = stats - # default values - self.host_regex = re.compile('') # allow all by default - self.domains_seen = set() @classmethod def from_crawler(cls, crawler): @@ -55,7 +52,7 @@ class OffsiteMiddleware(object): """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, 'allowed_domains', None) if not allowed_domains: - return + return re.compile('') # allow all by default url_pattern = re.compile("^https?://.*$") for domain in allowed_domains: if url_pattern.match(domain): @@ -65,9 +62,8 @@ class OffsiteMiddleware(object): return re.compile(regex) def spider_opened(self, spider): - host_regex = self.get_host_regex(spider) - if host_regex: - self.host_regex = host_regex + self.host_regex = self.get_host_regex(spider) + self.domains_seen = set() class URLWarning(Warning): From b8e8922d5436247f7be66c40e1a16a0acab7986e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 15 Jul 2018 17:50:55 -0300 Subject: [PATCH 020/140] Simplify stuff. Add more tests. --- scrapy/core/spidermw.py | 10 +++--- tests/test_spidermiddleware.py | 63 ++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 8ee42c2cf..c733402b9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -75,8 +75,8 @@ class SpiderMiddlewareManager(MiddlewareManager): # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for i, method in enumerate(self.methods['process_spider_exception']): - if i < index or method is None: + for method in self.methods['process_spider_exception'][index:]: + if method is None: continue result = method(response=response, exception=exception, spider=spider) index += 1 @@ -101,13 +101,13 @@ class SpiderMiddlewareManager(MiddlewareManager): for r in result_iterable: yield r except Exception as ex: - exception_result = process_spider_exception(Failure(ex), index) + exception_result = process_spider_exception(Failure(ex), index+1) if exception_result is None or isinstance(exception_result, Failure): raise recovered.extend(exception_result) - for i, method in enumerate(self.methods['process_spider_output']): - if i < index or method is None: + for method in self.methods['process_spider_output'][index:]: + if method is None: continue result = method(response=response, result=result, spider=spider) index += 1 diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index c33eb28ca..645d95059 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -113,6 +113,49 @@ class NotAGeneratorCallbackSpider(Spider): return [{'test': 1}, {'test': 1/0}] +# ================================================================================ +# (4) exceptions from a middleware process_spider_output method (generator) +class GeneratorOutputChainSpider(Spider): + name = 'GeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.GeneratorFailOutputChainMiddleware': 10, + __name__ + '.GeneratorRecoverOutputChainMiddleware': 5, + }, + } + + def start_requests(self): + yield Request(self.mockserver.url('/status?n=200')) + + def parse(self, response): + yield {'processed': ['parse']} + + +class GeneratorFailOutputChainMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + yield r + raise LookupError() + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + yield {'processed': [method]} + + +class GeneratorRecoverOutputChainMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + yield r + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + yield {'processed': [method]} + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod @@ -162,7 +205,8 @@ class TestSpiderMiddleware(TestCase): def test_generator_callback(self): """ (2) An exception from a spider callback (returning a generator) should - be caught by the process_spider_exception chain + be caught by the process_spider_exception chain. Items yielded before the + exception is raised should be processed normally. """ log2 = yield self.crawl_log(GeneratorCallbackSpider) self.assertIn("Middleware: ImportError exception caught", str(log2)) @@ -172,8 +216,23 @@ class TestSpiderMiddleware(TestCase): def test_not_a_generator_callback(self): """ (3) An exception from a spider callback (returning a list) should - be caught by the process_spider_exception chain + be caught by the process_spider_exception chain. No items should be processed. """ log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) self.assertNotIn("item_scraped_count", str(log3)) + + @defer.inlineCallbacks + def test_generator_output_chain(self): + """ + (4) An exception from a middleware's process_spider_output method should be sent + to the process_spider_exception method from the next middleware in the chain. + The final item count should be 2 (one from the spider callback and one from the + process_spider_exception chain) + """ + log4 = yield self.crawl_log(GeneratorOutputChainSpider) + self.assertIn("'item_scraped_count': 2", str(log4)) + self.assertIn("GeneratorRecoverOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertNotIn("GeneratorFailOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn("{'processed': ['parse', 'GeneratorFailOutputChainMiddleware.process_spider_output', 'GeneratorRecoverOutputChainMiddleware.process_spider_output']}", str(log4)) + self.assertIn("{'processed': ['GeneratorRecoverOutputChainMiddleware.process_spider_exception']}", str(log4)) From 56e92d90fda3e812aca270327e513391591a10cc Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 17 Jul 2018 15:15:38 -0300 Subject: [PATCH 021/140] Update tests --- tests/test_spidermiddleware.py | 54 ++++++++++++++++++++++++++++------ 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 645d95059..9bb7f62fd 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -119,8 +119,10 @@ class GeneratorOutputChainSpider(Spider): name = 'GeneratorOutputChainSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { - __name__ + '.GeneratorFailOutputChainMiddleware': 10, - __name__ + '.GeneratorRecoverOutputChainMiddleware': 5, + __name__ + '.GeneratorFailMiddleware': 10, + __name__ + '.GeneratorDoNothingAfterFailureMiddleware': 8, + __name__ + '.GeneratorRecoverMiddleware': 5, + __name__ + '.GeneratorDoNothingAfterRecoveryMiddleware': 3, }, } @@ -128,10 +130,23 @@ class GeneratorOutputChainSpider(Spider): yield Request(self.mockserver.url('/status?n=200')) def parse(self, response): - yield {'processed': ['parse']} + yield {'processed': ['parse-first-item']} + yield {'processed': ['parse-second-item']} -class GeneratorFailOutputChainMiddleware: +class _GeneratorDoNothingMiddleware: + def process_spider_output(self, response, result, spider): + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + yield r + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return None + + +class GeneratorFailMiddleware: def process_spider_output(self, response, result, spider): for r in result: r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) @@ -144,7 +159,11 @@ class GeneratorFailOutputChainMiddleware: yield {'processed': [method]} -class GeneratorRecoverOutputChainMiddleware: +class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): + pass + + +class GeneratorRecoverMiddleware: def process_spider_output(self, response, result, spider): for r in result: r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) @@ -155,6 +174,9 @@ class GeneratorRecoverOutputChainMiddleware: logging.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} +class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): + pass + # ================================================================================ class TestSpiderMiddleware(TestCase): @@ -227,12 +249,26 @@ class TestSpiderMiddleware(TestCase): """ (4) An exception from a middleware's process_spider_output method should be sent to the process_spider_exception method from the next middleware in the chain. + The result of the recovery by the process_spider_exception method should be handled + by the process_spider_output method from the next middleware. The final item count should be 2 (one from the spider callback and one from the process_spider_exception chain) """ log4 = yield self.crawl_log(GeneratorOutputChainSpider) self.assertIn("'item_scraped_count': 2", str(log4)) - self.assertIn("GeneratorRecoverOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertNotIn("GeneratorFailOutputChainMiddleware.process_spider_exception: LookupError caught", str(log4)) - self.assertIn("{'processed': ['parse', 'GeneratorFailOutputChainMiddleware.process_spider_output', 'GeneratorRecoverOutputChainMiddleware.process_spider_output']}", str(log4)) - self.assertIn("{'processed': ['GeneratorRecoverOutputChainMiddleware.process_spider_exception']}", str(log4)) + self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: LookupError caught", str(log4)) + self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught", str(log4)) + item_from_callback = {'processed': [ + 'parse-first-item', + 'GeneratorFailMiddleware.process_spider_output', + 'GeneratorDoNothingAfterFailureMiddleware.process_spider_output', + 'GeneratorRecoverMiddleware.process_spider_output', + 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + item_recovered = {'processed': [ + 'GeneratorRecoverMiddleware.process_spider_exception', + 'GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + self.assertIn(str(item_from_callback), str(log4)) + self.assertIn(str(item_recovered), str(log4)) + self.assertNotIn('parse-second-item', str(log4)) From 610f589662ca9c5929527e15cad2c347c8d5d335 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 17 Jul 2018 19:13:03 -0300 Subject: [PATCH 022/140] Add callback and errback in the same step --- scrapy/core/spidermw.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index c733402b9..da51bc974 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -70,7 +70,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure, index): + def process_spider_exception(_failure, index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): @@ -91,7 +91,7 @@ class SpiderMiddlewareManager(MiddlewareManager): .format(fname(method), type(result))) return _failure - def process_spider_output(result, index): + def process_spider_output(result, index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -120,8 +120,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return chain(result, recovered) dfd = mustbe_deferred(process_spider_input, response) - dfd.addErrback(process_spider_exception, index=0) - dfd.addCallback(process_spider_output, index=0) + dfd.addCallbacks(callback=process_spider_output, errback=process_spider_exception) return dfd def process_start_requests(self, start_requests, spider): From a3af0bfd56770aab0a056ae6e29efffa8b7d88c4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Jul 2018 15:15:55 -0300 Subject: [PATCH 023/140] More tests --- scrapy/core/spidermw.py | 21 +++++--- tests/test_spidermiddleware.py | 99 ++++++++++++++++++++++++++++++++-- 2 files changed, 110 insertions(+), 10 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index da51bc974..96488806d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -64,8 +64,8 @@ class SpiderMiddlewareManager(MiddlewareManager): try: result = method(response=response, spider=spider) if result is not None: - raise _InvalidOutput('Middleware {} must return None or raise ' \ - 'an exception, got {}'.format(fname(method), type(result))) + raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \ + .format(fname(method), type(result))) except: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) @@ -78,8 +78,8 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods['process_spider_exception'][index:]: if method is None: continue - result = method(response=response, exception=exception, spider=spider) index += 1 + result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned @@ -96,9 +96,9 @@ class SpiderMiddlewareManager(MiddlewareManager): # chain, they went through it already from the process_spider_exception method recovered = MutableChain() - def evaluate_result(result_iterable, index): + def evaluate_iterable(iterable, index): try: - for r in result_iterable: + for r in iterable: yield r except Exception as ex: exception_result = process_spider_exception(Failure(ex), index+1) @@ -109,10 +109,17 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods['process_spider_output'][index:]: if method is None: continue - result = method(response=response, result=result, spider=spider) index += 1 + # the following might fail directly if the output value is not a generator + try: + result = method(response=response, result=result, spider=spider) + except Exception as ex: + exception_result = process_spider_exception(Failure(ex), index+1) + if exception_result is None or isinstance(exception_result, Failure): + raise + return exception_result if _isiterable(result): - result = evaluate_result(result, index) + result = evaluate_iterable(result, index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 9bb7f62fd..2f431ddc7 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -98,8 +98,8 @@ class GeneratorCallbackSpider(Spider): # ================================================================================ # (3) exceptions from a spider callback (not a generator) -class NotAGeneratorCallbackSpider(Spider): - name = 'NotAGeneratorCallbackSpider' +class NotGeneratorCallbackSpider(Spider): + name = 'NotGeneratorCallbackSpider' custom_settings = { 'SPIDER_MIDDLEWARES': { __name__ + '.LogExceptionMiddleware': 10, @@ -178,6 +178,76 @@ class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): pass +# ================================================================================ +# (5) exceptions from a middleware process_spider_output method (not generator) +class NotGeneratorOutputChainSpider(Spider): + name = 'NotGeneratorOutputChainSpider' + custom_settings = { + 'SPIDER_MIDDLEWARES': { + __name__ + '.NotGeneratorFailMiddleware': 10, + __name__ + '.NotGeneratorDoNothingAfterFailureMiddleware': 8, + __name__ + '.NotGeneratorRecoverMiddleware': 5, + __name__ + '.NotGeneratorDoNothingAfterRecoveryMiddleware': 3, + }, + } + + def start_requests(self): + return [Request(self.mockserver.url('/status?n=200'))] + + def parse(self, response): + return [{'processed': ['parse-first-item']}, {'processed': ['parse-second-item']}] + + +class _NotGeneratorDoNothingMiddleware: + def process_spider_output(self, response, result, spider): + out = [] + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + out.append(r) + return out + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return None + + +class NotGeneratorFailMiddleware: + def process_spider_output(self, response, result, spider): + out = [] + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + out.append(r) + raise ReferenceError() + return out + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return [{'processed': [method]}] + + +class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware): + pass + + +class NotGeneratorRecoverMiddleware: + def process_spider_output(self, response, result, spider): + out = [] + for r in result: + r['processed'].append('{}.process_spider_output'.format(self.__class__.__name__)) + out.append(r) + return out + + def process_spider_exception(self, response, exception, spider): + method = '{}.process_spider_exception'.format(self.__class__.__name__) + logging.info('%s: %s caught', method, exception.__class__.__name__) + return [{'processed': [method]}] + +class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): + pass + + # ================================================================================ class TestSpiderMiddleware(TestCase): @classmethod @@ -240,7 +310,7 @@ class TestSpiderMiddleware(TestCase): (3) An exception from a spider callback (returning a list) should be caught by the process_spider_exception chain. No items should be processed. """ - log3 = yield self.crawl_log(NotAGeneratorCallbackSpider) + log3 = yield self.crawl_log(NotGeneratorCallbackSpider) self.assertIn("Middleware: ZeroDivisionError exception caught", str(log3)) self.assertNotIn("item_scraped_count", str(log3)) @@ -272,3 +342,26 @@ class TestSpiderMiddleware(TestCase): self.assertIn(str(item_from_callback), str(log4)) self.assertIn(str(item_recovered), str(log4)) self.assertNotIn('parse-second-item', str(log4)) + + @defer.inlineCallbacks + def test_not_a_generator_output_chain(self): + """ + (5) An exception from a middleware's process_spider_output method should be sent + to the process_spider_exception method from the next middleware in the chain. + The result of the recovery by the process_spider_exception method should be handled + by the process_spider_output method from the next middleware. + The final item count should be 1 (from the process_spider_exception chain, the items + from the spider callback are lost) + """ + log5 = yield self.crawl_log(NotGeneratorOutputChainSpider) + self.assertIn("'item_scraped_count': 1", str(log5)) + self.assertIn("GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertIn("GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn("GeneratorFailMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + self.assertNotIn("GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught", str(log5)) + item_recovered = {'processed': [ + 'NotGeneratorRecoverMiddleware.process_spider_exception', + 'NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output']} + self.assertIn(str(item_recovered), str(log5)) + self.assertNotIn('parse-first-item', str(log5)) + self.assertNotIn('parse-second-item', str(log5)) From 6329441c826bec97aeec82d3e7ec0bcd91c60a47 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Jul 2018 16:59:24 -0300 Subject: [PATCH 024/140] ModuleNotFoundError was added in py3.6 --- tests/test_spidermiddleware.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2f431ddc7..0451dfd27 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -34,7 +34,7 @@ class RecoverySpider(Spider): yield {'test': 1} self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) if not response.meta.get('dont_fail'): - raise ModuleNotFoundError() + raise TabError() class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): @@ -275,8 +275,8 @@ class TestSpiderMiddleware(TestCase): was enqueued from the recovery middleware) """ log = yield self.crawl_log(RecoverySpider) - self.assertIn("Middleware: ModuleNotFoundError exception caught", str(log)) - self.assertEqual(str(log).count("Middleware: ModuleNotFoundError exception caught"), 1) + self.assertIn("Middleware: TabError exception caught", str(log)) + self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1) self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks From 71a1406c99e7d4cced0693389e537c98a38104aa Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 18 Jul 2018 17:40:30 -0300 Subject: [PATCH 025/140] Logging changes --- tests/test_spidermiddleware.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 0451dfd27..0f5646a72 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,6 +1,4 @@ -import logging - from testfixtures import LogCapture from twisted.trial.unittest import TestCase from twisted.internet import defer @@ -13,7 +11,7 @@ from tests.spiders import MockServerSpider class LogExceptionMiddleware: def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) return None @@ -32,13 +30,13 @@ class RecoverySpider(Spider): def parse(self, response): yield {'test': 1} - self.logger.warn('DONT_FAIL: %s', response.meta.get('dont_fail')) + self.logger.info('DONT_FAIL: %s', response.meta.get('dont_fail')) if not response.meta.get('dont_fail'): raise TabError() class RecoveryMiddleware: def process_spider_exception(self, response, exception, spider): - logging.warn('Middleware: %s exception caught', exception.__class__.__name__) + spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__) return [ {'from': 'process_spider_exception'}, Request(response.url, meta={'dont_fail': True}, dont_filter=True), @@ -67,13 +65,13 @@ class ProcessSpiderInputSpider(Spider): return {'from': 'callback'} def errback(self, failure): - self.logger.warn('Got a Failure on the Request errback') + self.logger.info('Got a Failure on the Request errback') return {'from': 'errback'} class FailProcessSpiderInputMiddleware: def process_spider_input(self, response, spider): - logging.warn('Middleware: will raise IndexError') + spider.logger.info('Middleware: will raise IndexError') raise IndexError() @@ -142,7 +140,7 @@ class _GeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -155,7 +153,7 @@ class GeneratorFailMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} @@ -171,7 +169,7 @@ class GeneratorRecoverMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) yield {'processed': [method]} class GeneratorDoNothingAfterRecoveryMiddleware(_GeneratorDoNothingMiddleware): @@ -208,7 +206,7 @@ class _NotGeneratorDoNothingMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return None @@ -223,7 +221,7 @@ class NotGeneratorFailMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] @@ -241,7 +239,7 @@ class NotGeneratorRecoverMiddleware: def process_spider_exception(self, response, exception, spider): method = '{}.process_spider_exception'.format(self.__class__.__name__) - logging.info('%s: %s caught', method, exception.__class__.__name__) + spider.logger.info('%s: %s caught', method, exception.__class__.__name__) return [{'processed': [method]}] class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): From 20defa2e16628b4b432a1cc44ad37182dfc764ee Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 19 Jul 2018 10:31:06 -0300 Subject: [PATCH 026/140] Better handling of method indexes --- scrapy/core/spidermw.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 96488806d..8607ed620 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -70,20 +70,19 @@ class SpiderMiddlewareManager(MiddlewareManager): return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) - def process_spider_exception(_failure, index=0): + def process_spider_exception(_failure, start_index=0): exception = _failure.value # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for method in self.methods['process_spider_exception'][index:]: + for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index): if method is None: continue - index += 1 result = method(response=response, exception=exception, spider=spider) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - return process_spider_output(result, index) + return process_spider_output(result, method_index+1) elif result is None: continue else: @@ -91,7 +90,7 @@ class SpiderMiddlewareManager(MiddlewareManager): .format(fname(method), type(result))) return _failure - def process_spider_output(result, index=0): + def process_spider_output(result, start_index=0): # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method recovered = MutableChain() @@ -106,20 +105,19 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recovered.extend(exception_result) - for method in self.methods['process_spider_output'][index:]: + for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index): if method is None: continue - index += 1 # the following might fail directly if the output value is not a generator try: result = method(response=response, result=result, spider=spider) except Exception as ex: - exception_result = process_spider_exception(Failure(ex), index+1) + exception_result = process_spider_exception(Failure(ex), method_index+1) if exception_result is None or isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): - result = evaluate_iterable(result, index) + result = evaluate_iterable(result, method_index) else: raise _InvalidOutput('Middleware {} must return an iterable, got {}' \ .format(fname(method), type(result))) From 784eed113021a1a787787a354add242e7abbf6f9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 20 Jul 2018 19:08:46 -0300 Subject: [PATCH 027/140] Improve test coverage (downloader middleware) --- scrapy/core/downloader/middleware.py | 6 +-- tests/test_downloadermiddleware.py | 64 ++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index cf0c1f869..2fa277e7d 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -50,8 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): defer.returnValue(response) for method in self.methods['process_response']: - response = yield method(request=request, response=response, - spider=spider) + response = yield method(request=request, response=response, spider=spider) if not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \ (six.get_method_self(method).__class__.__name__, type(response))) @@ -63,8 +62,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): def process_exception(_failure): exception = _failure.value for method in self.methods['process_exception']: - response = yield method(request=request, exception=exception, - spider=spider) + response = yield method(request=request, exception=exception, spider=spider) if response is not None and not isinstance(response, (Response, Request)): raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \ (six.get_method_self(method).__class__.__name__, type(response))) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index fb51392b2..0f420b70d 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -3,6 +3,7 @@ from twisted.python.failure import Failure from scrapy.http import Request, Response from scrapy.spiders import Spider +from scrapy.exceptions import _InvalidOutput from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.utils.test import get_crawler from scrapy.utils.python import to_bytes @@ -115,3 +116,66 @@ class ResponseFromProcessRequestTest(ManagerTestCase): self.assertIs(results[0], resp) self.assertFalse(download_func.called) + + +class ProcessRequestInvalidOutput(ManagerTestCase): + """Invalid return value for process_request method should raise an exception""" + + def test_invalid_process_request(self): + req = Request('http://example.com/index.html') + resp = Response('http://example.com/index.html') + + class InvalidProcessRequestMiddleware: + def process_request(self, request, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessRequestMiddleware()) + download_func = mock.MagicMock() + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self.assertIsInstance(results[0], Failure) + self.assertIsInstance(results[0].value, _InvalidOutput) + + +class ProcessResponseInvalidOutput(ManagerTestCase): + """Invalid return value for process_response method should raise an exception""" + + def test_invalid_process_response(self): + req = Request('http://example.com/index.html') + resp = Response('http://example.com/index.html') + + class InvalidProcessResponseMiddleware: + def process_response(self, request, response, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessResponseMiddleware()) + download_func = mock.MagicMock() + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self.assertIsInstance(results[0], Failure) + self.assertIsInstance(results[0].value, _InvalidOutput) + + +class ProcessExceptionInvalidOutput(ManagerTestCase): + """Invalid return value for process_exception method should raise an exception""" + + def test_invalid_process_exception(self): + req = Request('http://example.com/index.html') + resp = Response('http://example.com/index.html') + + class InvalidProcessExceptionMiddleware: + def process_request(self, request, spider): + raise Exception() + + def process_exception(self, request, exception, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessExceptionMiddleware()) + download_func = mock.MagicMock() + dfd = self.mwman.download(download_func, req, self.spider) + results = [] + dfd.addBoth(results.append) + self.assertIsInstance(results[0], Failure) + self.assertIsInstance(results[0].value, _InvalidOutput) From d6d3e87e3a4fd306829a84f934a971fd4e337a26 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 27 Jul 2018 14:47:52 -0300 Subject: [PATCH 028/140] Rename test file --- ..._spidermiddleware.py => test_spidermiddleware_output_chain.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_spidermiddleware.py => test_spidermiddleware_output_chain.py} (100%) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware_output_chain.py similarity index 100% rename from tests/test_spidermiddleware.py rename to tests/test_spidermiddleware_output_chain.py From 801d3c07b4b7e57d50429e714a8255e7747568f4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 27 Jul 2018 15:06:25 -0300 Subject: [PATCH 029/140] Fix bad exception handling, add tests --- scrapy/core/spidermw.py | 2 + tests/test_spidermiddleware_invalid_values.py | 82 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 tests/test_spidermiddleware_invalid_values.py diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 8607ed620..1b67af130 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -66,6 +66,8 @@ class SpiderMiddlewareManager(MiddlewareManager): if result is not None: raise _InvalidOutput('Middleware {} must return None or raise an exception, got {}' \ .format(fname(method), type(result))) + except _InvalidOutput: + raise except: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware_invalid_values.py new file mode 100644 index 000000000..0d9af8951 --- /dev/null +++ b/tests/test_spidermiddleware_invalid_values.py @@ -0,0 +1,82 @@ +from twisted.trial.unittest import TestCase +from twisted.python.failure import Failure + +from scrapy.spiders import Spider +from scrapy.http import Request, Response +from scrapy.exceptions import _InvalidOutput +from scrapy.utils.test import get_crawler +from scrapy.core.spidermw import SpiderMiddlewareManager +from tests import mock + + +class SpiderMiddlewareTestCase(TestCase): + + def setUp(self): + self.request = Request('http://example.com/index.html') + self.response = Response(self.request.url, request=self.request) + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler) + + def _scrape_response(self): + """Execute spider mw manager's scrape_response method and return the result. + Raise exception in case of failure. + """ + scrape_func = mock.MagicMock() + dfd = self.mwman.scrape_response(scrape_func, self.response, self.request, self.spider) + # catch deferred result and return the value + results = [] + dfd.addBoth(results.append) + self._wait(dfd) + ret = results[0] + return ret + + +class ProcessSpiderInputInvalidOutput(SpiderMiddlewareTestCase): + """Invalid return value for process_spider_input method""" + + def test_invalid_process_spider_input(self): + + class InvalidProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, _InvalidOutput) + + +class ProcessSpiderOutputInvalidOutput(SpiderMiddlewareTestCase): + """Invalid return value for process_spider_output method""" + + def test_invalid_process_spider_output(self): + + class InvalidProcessSpiderOutputMiddleware: + def process_spider_output(self, response, result, spider): + return 1 + + self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, _InvalidOutput) + + +class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): + """Invalid return value for process_spider_exception method""" + + def test_invalid_process_spider_exception(self): + + class InvalidProcessSpiderOutputExceptionMiddleware: + def process_spider_exception(self, response, exception, spider): + return 1 + + class RaiseExceptionProcessSpiderOutputMiddleware: + def process_spider_output(self, response, result, spider): + raise Exception() + + self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) + self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, _InvalidOutput) From 8c55f5eb159ae85d468a89b74ffaff3e824144ab Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Aug 2018 15:16:26 -0300 Subject: [PATCH 030/140] Simplify check for re-raised exception. Add tests. --- scrapy/core/spidermw.py | 4 ++-- ...lid_values.py => test_spidermiddleware.py} | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) rename tests/{test_spidermiddleware_invalid_values.py => test_spidermiddleware.py} (79%) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 1b67af130..4268c91d6 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -103,7 +103,7 @@ class SpiderMiddlewareManager(MiddlewareManager): yield r except Exception as ex: exception_result = process_spider_exception(Failure(ex), index+1) - if exception_result is None or isinstance(exception_result, Failure): + if isinstance(exception_result, Failure): raise recovered.extend(exception_result) @@ -115,7 +115,7 @@ class SpiderMiddlewareManager(MiddlewareManager): result = method(response=response, result=result, spider=spider) except Exception as ex: exception_result = process_spider_exception(Failure(ex), method_index+1) - if exception_result is None or isinstance(exception_result, Failure): + if isinstance(exception_result, Failure): raise return exception_result if _isiterable(result): diff --git a/tests/test_spidermiddleware_invalid_values.py b/tests/test_spidermiddleware.py similarity index 79% rename from tests/test_spidermiddleware_invalid_values.py rename to tests/test_spidermiddleware.py index 0d9af8951..54756f2ff 100644 --- a/tests/test_spidermiddleware_invalid_values.py +++ b/tests/test_spidermiddleware.py @@ -80,3 +80,23 @@ class ProcessSpiderExceptionInvalidOutput(SpiderMiddlewareTestCase): result = self._scrape_response() self.assertIsInstance(result, Failure) self.assertIsInstance(result.value, _InvalidOutput) + + +class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): + """Re raise the exception by returning None""" + + def test_process_spider_exception_return_none(self): + + class ProcessSpiderOutputExceptionReturnNoneMiddleware: + def process_spider_exception(self, response, exception, spider): + return None + + class RaiseExceptionProcessSpiderOutputMiddleware: + def process_spider_output(self, response, result, spider): + 1/0 + + self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware()) + self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + result = self._scrape_response() + self.assertIsInstance(result, Failure) + self.assertIsInstance(result.value, ZeroDivisionError) From 40449fa0eb707bac1ae2b78f0f812372e90f17b7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 3 Aug 2018 18:20:25 -0300 Subject: [PATCH 031/140] Update docs, add tests, remove FIXME comment --- docs/topics/spider-middleware.rst | 3 +- scrapy/core/scraper.py | 1 - tests/test_spidermiddleware.py | 4 +- tests/test_spidermiddleware_output_chain.py | 43 ++++++++++++++------- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 915833c54..7db623cf4 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -78,7 +78,8 @@ following methods: If it raises an exception, Scrapy won't bother calling any other spider middleware :meth:`process_spider_input` and will call the request - errback. The output of the errback is chained back in the other + errback if there is one, otherwise it will start the :meth:`process_spider_exception` + chain. The output of the errback is chained back in the other direction for :meth:`process_spider_output` to process it, or :meth:`process_spider_exception` if it raised an exception. diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index ee1e95a0c..d7fe721fb 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -135,7 +135,6 @@ class Scraper(object): return self.spidermw.scrape_response( self.call_spider, request_result, request, spider) else: - # FIXME: don't ignore errors in spider middleware dfd = self.call_spider(request_result, request, spider) return dfd.addErrback( self._log_download_errors, request_result, request, spider) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 54756f2ff..832fd3330 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -87,7 +87,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): def test_process_spider_exception_return_none(self): - class ProcessSpiderOutputExceptionReturnNoneMiddleware: + class ProcessSpiderExceptionReturnNoneMiddleware: def process_spider_exception(self, response, exception, spider): return None @@ -95,7 +95,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase): def process_spider_output(self, response, result, spider): 1/0 - self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware()) + self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) result = self._scrape_response() self.assertIsInstance(result, Failure) diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 0f5646a72..6f8727a15 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -45,8 +45,13 @@ class RecoveryMiddleware: # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method -class ProcessSpiderInputSpider(Spider): - name = 'ProcessSpiderInputSpider' +class FailProcessSpiderInputMiddleware: + def process_spider_input(self, response, spider): + spider.logger.info('Middleware: will raise IndexError') + raise IndexError() + +class ProcessSpiderInputSpiderWithoutErrback(Spider): + name = 'ProcessSpiderInputSpiderWithoutErrback' custom_settings = { 'SPIDER_MIDDLEWARES': { # spider @@ -58,23 +63,23 @@ class ProcessSpiderInputSpider(Spider): } def start_requests(self): - yield Request(url=self.mockserver.url('/status?n=200'), - callback=self.parse, errback=self.errback) + yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse) def parse(self, response): return {'from': 'callback'} + +class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback): + name = 'ProcessSpiderInputSpiderWithErrback' + + def start_requests(self): + yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback) + def errback(self, failure): self.logger.info('Got a Failure on the Request errback') return {'from': 'errback'} -class FailProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): - spider.logger.info('Middleware: will raise IndexError') - raise IndexError() - - # ================================================================================ # (2) exceptions from a spider callback (generator) class GeneratorCallbackSpider(Spider): @@ -278,12 +283,22 @@ class TestSpiderMiddleware(TestCase): self.assertIn("'item_scraped_count': 3", str(log)) @defer.inlineCallbacks - def test_process_spider_input_errback(self): + def test_process_spider_input_without_errback(self): """ - (1) An exception from the process_spider_input chain should not be caught by the - process_spider_exception chain, it should go directly to the Request errback + (1.1) An exception from the process_spider_input chain should be caught by the + process_spider_exception chain from the start if the Request has no errback """ - log1 = yield self.crawl_log(ProcessSpiderInputSpider) + log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback) + self.assertIn("Middleware: will raise IndexError", str(log1)) + self.assertIn("Middleware: IndexError exception caught", str(log1)) + + @defer.inlineCallbacks + def test_process_spider_input_with_errback(self): + """ + (1.2) An exception from the process_spider_input chain should not be caught by the + process_spider_exception chain if the Request has an errback + """ + log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback) self.assertNotIn("Middleware: IndexError exception caught", str(log1)) self.assertIn("Middleware: will raise IndexError", str(log1)) self.assertIn("Got a Failure on the Request errback", str(log1)) From 58f5565357ed532970772cd55c2d17d1e00198a9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:23:12 -0300 Subject: [PATCH 033/140] Move MutableChain to scrapy.utils.python --- scrapy/core/spidermw.py | 21 +-------------------- scrapy/utils/python.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4268c91d6..d776430e5 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -11,32 +11,13 @@ from scrapy.exceptions import _InvalidOutput from scrapy.middleware import MiddlewareManager from scrapy.utils.defer import mustbe_deferred from scrapy.utils.conf import build_component_list +from scrapy.utils.python import MutableChain def _isiterable(possible_iterator): return hasattr(possible_iterator, '__iter__') -class MutableChain: - """ - Thin wrapper around itertools.chain, allowing to add iterables "in-place" - """ - def __init__(self, *args): - self.data = chain(*args) - - def extend(self, *iterables): - self.data = chain(self.data, *iterables) - - def __iter__(self): - return self.data.__iter__() - - def __next__(self): # py3 - return self.data.__next__() - - def next(self): # py2 - return self.data.next() - - class SpiderMiddlewareManager(MiddlewareManager): component_name = 'spider middleware' diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 732ca13a0..7971b4dde 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -9,6 +9,7 @@ import weakref import errno import six from functools import partial, wraps +from itertools import chain import sys from scrapy.utils.decorators import deprecated @@ -387,3 +388,23 @@ if hasattr(sys, "pypy_version_info"): else: def garbage_collect(): gc.collect() + + +class MutableChain(object): + """ + Thin wrapper around itertools.chain, allowing to add iterables "in-place" + """ + def __init__(self, *args): + self.data = chain(*args) + + def extend(self, *iterables): + self.data = chain(self.data, *iterables) + + def __iter__(self): + return self.data.__iter__() + + def __next__(self): # py3 + return self.data.__next__() + + def next(self): # py2 + return self.data.next() From a05eaeed73a469493e78b5a1c5f0b4de2adf41c2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:31:51 -0300 Subject: [PATCH 034/140] Simplify MutableChain --- scrapy/utils/python.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7971b4dde..1a6bab990 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -403,8 +403,7 @@ class MutableChain(object): def __iter__(self): return self.data.__iter__() - def __next__(self): # py3 - return self.data.__next__() + def __next__(self): + return next(self.data) - def next(self): # py2 - return self.data.next() + next = __next__ From 15f0a890ee9f059111333fdeb6c6c3b5a8dadc07 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:34:59 -0300 Subject: [PATCH 035/140] Assign processing methods to a variable before iterating --- scrapy/core/spidermw.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d776430e5..3fae770a9 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -58,7 +58,8 @@ class SpiderMiddlewareManager(MiddlewareManager): # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - for method_index, method in enumerate(self.methods['process_spider_exception'][start_index:], start=start_index): + method_list = self.methods['process_spider_exception'][start_index:] + for method_index, method in enumerate(method_list, start=start_index): if method is None: continue result = method(response=response, exception=exception, spider=spider) @@ -88,7 +89,8 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recovered.extend(exception_result) - for method_index, method in enumerate(self.methods['process_spider_output'][start_index:], start=start_index): + method_list = self.methods['process_spider_output'][start_index:] + for method_index, method in enumerate(method_list, start=start_index): if method is None: continue # the following might fail directly if the output value is not a generator From e0360e5223b618934ee006b4c9ed63012e7e621f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 11 Oct 2018 11:55:13 -0300 Subject: [PATCH 036/140] Add tests for MutableChain --- tests/test_utils_python.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index f6133657b..3e1148354 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,11 +9,23 @@ import six from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, WeakKeyCache, stringify_dict, get_func_args, to_bytes, to_unicode, - without_none_values) + without_none_values, MutableChain) __doctests__ = ['scrapy.utils.python'] +class MutableChainTest(unittest.TestCase): + def test_mutablechain(self): + m = MutableChain(range(2), [2, 3], (4, 5)) + m.extend(range(6, 7)) + m.extend([7, 8]) + m.extend([9, 10], (11, 12)) + self.assertEqual(next(m), 0) + self.assertEqual(m.next(), 1) + self.assertEqual(m.__next__(), 2) + self.assertEqual(list(m), list(range(3, 13))) + + class ToUnicodeTest(unittest.TestCase): def test_converting_an_utf8_encoded_string_to_unicode(self): self.assertEqual(to_unicode(b'lel\xc3\xb1e'), u'lel\xf1e') From a25cf5c82f99f7ae11346a2e565d6255835c3814 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 20 Nov 2018 16:13:09 +0000 Subject: [PATCH 038/140] function to get unique file queues for any type of base queue --- scrapy/core/queues.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 scrapy/core/queues.py diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py new file mode 100644 index 000000000..96d582fc7 --- /dev/null +++ b/scrapy/core/queues.py @@ -0,0 +1,15 @@ +import uuid +import os.path + + +def unique_files_queue(queue_class): + + class UniqueFilesQueue(queue_class): + def __init__(self, path): + path = path + "-" + uuid.uuid4().hex + while os.path.exists(path): + path = path + "-" + uuid.uuid4().hex + + super().__init__(path) + + return UniqueFilesQueue From 6c78b3d5ef94791b11c2ce3dfd5cebd757a68b2a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 3 Jan 2019 13:15:58 -0300 Subject: [PATCH 039/140] Deques can't be sliced, use itertools.islice instead --- scrapy/core/spidermw.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 58bd7c2c8..e07f76bdf 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -3,7 +3,7 @@ Spider Middleware manager See documentation in docs/topics/spider-middleware.rst """ -from itertools import chain +from itertools import chain, islice import six from twisted.python.failure import Failure @@ -58,7 +58,7 @@ class SpiderMiddlewareManager(MiddlewareManager): # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): return _failure - method_list = self.methods['process_spider_exception'][start_index:] + method_list = islice(self.methods['process_spider_exception'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue @@ -89,7 +89,7 @@ class SpiderMiddlewareManager(MiddlewareManager): raise recovered.extend(exception_result) - method_list = self.methods['process_spider_output'][start_index:] + method_list = islice(self.methods['process_spider_output'], start_index, None) for method_index, method in enumerate(method_list, start=start_index): if method is None: continue From e3e804cfb0fc05ef3fc569ec6e0af247ce504d06 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 28 Jan 2019 15:10:34 -0300 Subject: [PATCH 040/140] Styling nitpick :-) --- scrapy/core/downloader/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index a8e6f93a3..7a6a4dfac 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -41,7 +41,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): (six.get_method_self(method).__class__.__name__, response.__class__.__name__)) if response: defer.returnValue(response) - defer.returnValue((yield download_func(request=request,spider=spider))) + defer.returnValue((yield download_func(request=request, spider=spider))) @defer.inlineCallbacks def process_response(response): From 43fd6229684b3ccca564524fc92faf009a8c4c97 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 10:21:50 +0000 Subject: [PATCH 041/140] Rule.process_request: optionally take a Response object --- scrapy/spiders/crawl.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index e5ac72e18..5aec0fd83 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -24,12 +24,23 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request = process_request + self.process_request_function = process_request if follow is None: self.follow = False if callback else True else: self.follow = follow + def process_request(self, request, response): + """ + Wrapper around the request processing function to maintain backward compatibility + with functions that do not take a Response object as parameter. + """ + argcount = self.process_request_function.__code__.co_argcount + if getattr(self.process_request_function, '__self__', None): + argcount = argcount - 1 + args = [request] if argcount == 1 else [request, response] + return self.process_request_function(*args) + class CrawlSpider(Spider): @@ -65,7 +76,7 @@ class CrawlSpider(Spider): for link in links: seen.add(link) r = self._build_request(n, link) - yield rule.process_request(r) + yield rule.process_request(r, response) def _response_downloaded(self, response): rule = self._rules[response.meta['rule']] @@ -93,7 +104,7 @@ class CrawlSpider(Spider): for rule in self._rules: rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) - rule.process_request = get_method(rule.process_request) + rule.process_request_function = get_method(rule.process_request_function) @classmethod def from_crawler(cls, crawler, *args, **kwargs): From 22fda61d62a2b230b0e8588eabb0d71cb77141b7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 10:54:38 +0000 Subject: [PATCH 042/140] Rule.process_request: tests --- tests/test_spider.py | 98 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/test_spider.py b/tests/test_spider.py index fefdaa403..5e20e0d99 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -263,6 +263,104 @@ class CrawlSpiderTest(SpiderTest): 'http://example.org/about.html', 'http://example.org/nofollow.html']) + def test_process_request(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + def process_request_change_domain(request): + return request.replace(url=request.url.replace('.org', '.com')) + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request=process_request_change_domain), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) + + def test_process_request_with_response(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + def process_request_meta_response_class(request, response): + request.meta['response_class'] = response.__class__.__name__ + return request + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request=process_request_meta_response_class), + ) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) + self.assertEqual([r.meta['response_class'] for r in output], + ['HtmlResponse', 'HtmlResponse', 'HtmlResponse']) + + def test_process_request_instance_method(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request='process_request_upper'), + ) + + def process_request_upper(self, request): + return request.replace(url=request.url.upper()) + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + + def test_process_request_instance_method_with_response(self): + + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) + + class _CrawlSpider(self.spider_class): + name="test" + allowed_domains=['example.org'] + rules = ( + Rule(LinkExtractor(), process_request='process_request_meta_response_class'), + ) + + def process_request_meta_response_class(self, request, response): + request.meta['response_class'] = response.__class__.__name__ + return request + + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) + self.assertEqual([r.meta['response_class'] for r in output], + ['HtmlResponse', 'HtmlResponse', 'HtmlResponse']) + def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, 'example.com') From b30ca379b6785c7ceb75e12285fe7865b4f607d1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 11:02:51 +0000 Subject: [PATCH 043/140] Rule.process_request: docs --- docs/topics/spiders.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 742a88659..24b6f7ec9 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -402,10 +402,12 @@ Crawling rules of links extracted from each response using the specified ``link_extractor``. This is mainly used for filtering purposes. - ``process_request`` is a callable, or a string (in which case a method from - the spider object with that name will be used) which will be called with - every request extracted by this rule, and must return a request or None (to - filter out the request). + ``process_request`` is a callable (or a string, in which case a method from + the spider object with that name will be used) which will be called for + every request extracted by this rule. This callable should take a Request object + as first positional argument and, optionally, the Response object from which the + Request originated as second positional argument. It must return a request or None + (to filter out the request). CrawlSpider example ~~~~~~~~~~~~~~~~~~~ From 83ec947fe732035e147c21df352e199ce2cce5c8 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 13 Mar 2019 11:23:51 +0000 Subject: [PATCH 044/140] Rule.process_request defaults to None in the docs --- scrapy/spiders/crawl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 5aec0fd83..ad86fc19d 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -19,12 +19,12 @@ def identity(x): class Rule(object): - def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=identity): + def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): self.link_extractor = link_extractor self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request_function = process_request + self.process_request_function = process_request or identity if follow is None: self.follow = False if callback else True else: From 01ed605d02013b1d7955369562b2443d2a561599 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 15 Mar 2019 16:54:14 +0000 Subject: [PATCH 045/140] PEP8 changes to test_spider.py --- tests/test_spider.py | 60 +++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 5e20e0d99..c9af7a2d7 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -105,11 +105,11 @@ class SpiderTest(unittest.TestCase): def test_logger(self): spider = self.spider_class('example.com') - with LogCapture() as l: + with LogCapture() as lc: spider.logger.info('test log msg') - l.check(('example.com', 'INFO', 'test log msg')) + lc.check(('example.com', 'INFO', 'test log msg')) - record = l.records[0] + record = lc.records[0] self.assertIn('spider', record.__dict__) self.assertIs(record.spider, spider) @@ -190,12 +190,11 @@ class CrawlSpiderTest(SpiderTest): def test_process_links(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="dummy_process_links"), ) @@ -208,24 +207,24 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) def test_process_links_filter(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): import re - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="filter_process_links"), ) _test_regex = re.compile('nofollow') + def filter_process_links(self, links): return [link for link in links if not self._test_regex.search(link.url)] @@ -235,17 +234,16 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 2) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html']) def test_process_links_generator(self): - response = HtmlResponse("http://example.org/somepage/index.html", - body=self.test_body) + response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_links="dummy_process_links"), ) @@ -259,9 +257,9 @@ class CrawlSpiderTest(SpiderTest): self.assertEqual(len(output), 3) self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) self.assertEqual([r.url for r in output], - ['http://example.org/somepage/item/12.html', - 'http://example.org/about.html', - 'http://example.org/nofollow.html']) + ['http://example.org/somepage/item/12.html', + 'http://example.org/about.html', + 'http://example.org/nofollow.html']) def test_process_request(self): @@ -271,8 +269,8 @@ class CrawlSpiderTest(SpiderTest): return request.replace(url=request.url.replace('.org', '.com')) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request=process_request_change_domain), ) @@ -295,8 +293,8 @@ class CrawlSpiderTest(SpiderTest): return request class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request=process_request_meta_response_class), ) @@ -317,8 +315,8 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request='process_request_upper'), ) @@ -340,8 +338,8 @@ class CrawlSpiderTest(SpiderTest): response = HtmlResponse("http://example.org/somepage/index.html", body=self.test_body) class _CrawlSpider(self.spider_class): - name="test" - allowed_domains=['example.org'] + name = "test" + allowed_domains = ['example.org'] rules = ( Rule(LinkExtractor(), process_request='process_request_meta_response_class'), ) From 92bbc5290d2b381ea60d68442a887d1ba020874e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 16 Mar 2019 05:41:40 +0000 Subject: [PATCH 046/140] Rule.process_request - Renaming --- scrapy/spiders/crawl.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index ad86fc19d..c01f75798 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -24,22 +24,22 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request_function = process_request or identity + self.process_request = process_request or identity if follow is None: self.follow = False if callback else True else: self.follow = follow - def process_request(self, request, response): + def _process_request(self, request, response): """ Wrapper around the request processing function to maintain backward compatibility with functions that do not take a Response object as parameter. """ - argcount = self.process_request_function.__code__.co_argcount - if getattr(self.process_request_function, '__self__', None): + argcount = self.process_request.__code__.co_argcount + if hasattr(self.process_request, '__self__'): argcount = argcount - 1 args = [request] if argcount == 1 else [request, response] - return self.process_request_function(*args) + return self.process_request(*args) class CrawlSpider(Spider): @@ -75,8 +75,8 @@ class CrawlSpider(Spider): links = rule.process_links(links) for link in links: seen.add(link) - r = self._build_request(n, link) - yield rule.process_request(r, response) + request = self._build_request(n, link) + yield rule._process_request(request, response) def _response_downloaded(self, response): rule = self._rules[response.meta['rule']] @@ -104,7 +104,7 @@ class CrawlSpider(Spider): for rule in self._rules: rule.callback = get_method(rule.callback) rule.process_links = get_method(rule.process_links) - rule.process_request_function = get_method(rule.process_request_function) + rule.process_request = get_method(rule.process_request) @classmethod def from_crawler(cls, crawler, *args, **kwargs): From 821f5bb26077d7f9a6b2b1a72f210f81779f5393 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 3 Dec 2018 11:00:03 +0000 Subject: [PATCH 047/140] First implementation handle exception use O(N) instead of O(NlogN) here we have request as struct additional check for meptiness small performance improvement do not consume another request test number of responses mark requests back to 3 slots test case raise exceptions in case of missed meta add marks to requests and work only with your own requests only disk queue should obtain signals separate functions for slot rasd/write use signlas without variable stop crawler get signals in correct place logic test for download-aware priority queue update comment for structure ensure text type transform slot name to path use implicit structure use unicode type implicitly use real crawler add signals more slot accounting simple implementation of pop small slot accounting code no need for custom len function ability to call super in py27 add slots generic tests for downloader aware queue dummy implementation of crawler aware priority queue move common logic to base class rename class pass crawler to pqclass constructor do not copy quelib.PriorityQueue code add comment about new class remove obsolete function modify behaviour of queuelib.PriorityQueue to dodge very complex priority better way to get name remove obsolete commentary check boundaries function for priority convertion with known limits correct import path move file do not switch on by deffault as ip concurrency not supported set scheduler slot in case of empty slot use constant single place for added urls single place for constants use as default queue correct format for error text test migration from old version with on disk queue in these tests we have only two inflection points - jobdir and priority_queue_cls we do not need separate mock spider, use usual one do not rely on order of dict elements, imply order of list test round robiness of priority queue add comments and requirements for our magick function remove debug logging put queues into slot as we fabricate priorities we do not need special types anymore fabricate priority for priority queue more versatile priorities Scheduler class is not inflection point wrap correct types check for emptinees before initialization tests for new priority queue correct default type for startprios use exact values put common settings to base class test priorities for disk scheduler test dequeue for disk scheduler test length for disk scheduler setUp/tearDown methods for on disk schedulers new methods remove excessive line base class to handle scheduler creation correct method names test priorities deque test close scheduler on test end enqueue some requests test template for scheduler use downloader slot I/O implementation for RoundRobin queue round-robin implementation without I/O and slot detection wrappers for every disk queue class --- scrapy/core/downloader/__init__.py | 8 +- scrapy/core/queues.py | 15 -- scrapy/core/scheduler.py | 17 +- scrapy/pqueues.py | 246 ++++++++++++++++++++++ tests/test_scheduler.py | 315 +++++++++++++++++++++++++++++ 5 files changed, 578 insertions(+), 23 deletions(-) delete mode 100644 scrapy/core/queues.py create mode 100644 scrapy/pqueues.py create mode 100644 tests/test_scheduler.py diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 59c3ad074..4695d75f4 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -75,6 +75,8 @@ def _get_concurrency_delay(concurrency, spider, settings): class Downloader(object): + DOWNLOAD_SLOT = 'download_slot' + def __init__(self, crawler): self.settings = crawler.settings self.signals = crawler.signals @@ -111,8 +113,8 @@ class Downloader(object): return key, self.slots[key] def _get_slot_key(self, request, spider): - if 'download_slot' in request.meta: - return request.meta['download_slot'] + if self.DOWNLOAD_SLOT in request.meta: + return request.meta[self.DOWNLOAD_SLOT] key = urlparse_cached(request).hostname or '' if self.ip_concurrency: @@ -122,7 +124,7 @@ class Downloader(object): def _enqueue_request(self, request, spider): key, slot = self._get_slot(request, spider) - request.meta['download_slot'] = key + request.meta[self.DOWNLOAD_SLOT] = key def _deactivate(response): slot.active.remove(request) diff --git a/scrapy/core/queues.py b/scrapy/core/queues.py deleted file mode 100644 index 96d582fc7..000000000 --- a/scrapy/core/queues.py +++ /dev/null @@ -1,15 +0,0 @@ -import uuid -import os.path - - -def unique_files_queue(queue_class): - - class UniqueFilesQueue(queue_class): - def __init__(self, path): - path = path + "-" + uuid.uuid4().hex - while os.path.exists(path): - path = path + "-" + uuid.uuid4().hex - - super().__init__(path) - - return UniqueFilesQueue diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index eb790a67e..d40f3aa0c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) class Scheduler(object): def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, - logunser=False, stats=None, pqclass=None): + logunser=False, stats=None, pqclass=None, crawler=None): self.df = dupefilter self.dqdir = self._dqdir(jobdir) self.pqclass = pqclass @@ -21,6 +21,7 @@ class Scheduler(object): self.mqclass = mqclass self.logunser = logunser self.stats = stats + self.crawler = crawler @classmethod def from_crawler(cls, crawler): @@ -32,14 +33,15 @@ class Scheduler(object): mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG')) return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, - stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass) + stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, + mqclass=mqclass, crawler=crawler) def has_pending_requests(self): return len(self) > 0 def open(self, spider): self.spider = spider - self.mqs = self.pqclass(self._newmq) + self.mqs = create_instance(self.pqclass, None, self.crawler, self._newmq) self.dqs = self._dq() if self.dqdir else None return self.df.open() @@ -111,7 +113,7 @@ class Scheduler(object): return self.mqclass() def _newdq(self, priority): - return self.dqclass(join(self.dqdir, 'p%s' % priority)) + return self.dqclass(join(self.dqdir, 'p%s' % (priority, ))) def _dq(self): activef = join(self.dqdir, 'active.json') @@ -120,7 +122,12 @@ class Scheduler(object): prios = json.load(f) else: prios = () - q = self.pqclass(self._newdq, startprios=prios) + + q = create_instance(self.pqclass, + None, + self.crawler, + self._newdq, + startprios=prios) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", {'queuesize': len(q)}, extra={'spider': self.spider}) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py new file mode 100644 index 000000000..75073b7a4 --- /dev/null +++ b/scrapy/pqueues.py @@ -0,0 +1,246 @@ +from collections import deque +import hashlib +import logging +from six import text_type +from six.moves.urllib.parse import urlparse + +from queuelib import PriorityQueue + +from scrapy.core.downloader import Downloader +from scrapy.http import Request +from scrapy.signals import request_reached_downloader, response_downloaded + + +logger = logging.getLogger(__name__) + + +SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT + + +def _get_from_request(request, key, default=None): + if isinstance(request, dict): + return request.get(key, default) + + if isinstance(request, Request): + return getattr(request, key, default) + + raise ValueError('Bad type of request "%s"' % (request.__class__, )) + + +def _scheduler_slot_read(request, default=None): + meta = _get_from_request(request, 'meta', dict()) + slot = meta.get(SCHEDULER_SLOT_META_KEY, default) + return slot + + +def _scheduler_slot_write(request, slot): + meta = _get_from_request(request, 'meta', None) + if not isinstance(meta, dict): + raise ValueError('No meta attribute in %s' % (request, )) + meta[SCHEDULER_SLOT_META_KEY] = slot + + +def _scheduler_slot(request): + + slot = _scheduler_slot_read(request, None) + if slot is None: + url = _get_from_request(request, 'url') + slot = urlparse(url).hostname or '' + _scheduler_slot_write(request, slot) + + return slot + + +def _pathable(x): + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x]) + + """ + as we replace some letters we can get collision for different slots + add we add unique part + """ + unique_slot = hashlib.md5(x.encode('utf8')).hexdigest() + + return '-'.join([pathable_slot, unique_slot]) + + +class PrioritySlot: + __slots__ = ('priority', 'slot') + + def __init__(self, priority=0, slot=None): + self.priority = priority + self.slot = slot + + def __hash__(self): + return hash((self.priority, self.slot)) + + def __eq__(self, other): + return (self.priority, self.slot) == (other.priority, other.slot) + + def __lt__(self, other): + return (self.priority, self.slot) < (other.priority, other.slot) + + def __str__(self): + return '_'.join([text_type(self.priority), _pathable(text_type(self.slot))]) + + +class PriorityAsTupleQueue(PriorityQueue): + """ + Python structures is not directly (de)serialized (to)from json. + We need this modified queue to transform custom structure (from)to + json serializable structures + """ + def __init__(self, qfactory, startprios=()): + + super(PriorityAsTupleQueue, self).__init__( + qfactory, + [PrioritySlot(priority=p[0], slot=p[1]) for p in startprios] + ) + + def close(self): + startprios = super(PriorityAsTupleQueue, self).close() + return [(s.priority, s.slot) for s in startprios] + + def is_empty(self): + return not self.queues or len(self) == 0 + + +class SlotBasedPriorityQueue(object): + + def __init__(self, qfactory, startprios={}): + self.pqueues = dict() # slot -> priority queue + self.qfactory = qfactory # factory for creating new internal queues + + if not startprios: + return + + if not isinstance(startprios, dict): + raise ValueError("Looks like your priorities file malforfemed. " + "Possible reason: You run scrapy with previous " + "version. Interrupted it. Updated scrapy. And " + "run again.") + + for slot, prios in startprios.items(): + self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory, prios) + + def pop_slot(self, slot): + queue = self.pqueues[slot] + request = queue.pop() + is_empty = queue.is_empty() + if is_empty: + del self.pqueues[slot] + + return request, is_empty + + def push_slot(self, request, priority): + slot = _scheduler_slot(request) + is_new = False + if slot not in self.pqueues: + is_new = True + self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory) + self.pqueues[slot].push(request, PrioritySlot(priority=priority, slot=slot)) + return slot, is_new + + def close(self): + startprios = dict() + for slot, queue in self.pqueues.items(): + prios = queue.close() + startprios[slot] = prios + self.pqueues.clear() + return startprios + + def __len__(self): + return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + + +class RoundRobinPriorityQueue(SlotBasedPriorityQueue): + + def __init__(self, qfactory, startprios={}): + super(RoundRobinPriorityQueue, self).__init__(qfactory, startprios) + self._slots = deque() + for slot in self.pqueues: + self._slots.append(slot) + + def push(self, request, priority): + slot, is_new = self.push_slot(request, priority) + if is_new: + self._slots.append(slot) + + def pop(self): + if not self._slots: + return + + slot = self._slots.popleft() + request, is_empty = self.pop_slot(slot) + + if not is_empty: + self._slots.append(slot) + + return request + + def close(self): + self._slots.clear() + return super(RoundRobinPriorityQueue, self).close() + + +class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): + + _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' + + @classmethod + def from_crawler(cls, crawler, qfactory, startprios={}): + return cls(crawler, qfactory, startprios) + + def __init__(self, crawler, qfactory, startprios={}): + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) + self._slots = {slot: 0 for slot in self.pqueues} + crawler.signals.connect(self.on_response_download, + signal=response_downloaded) + crawler.signals.connect(self.on_request_reached_downloader, + signal=request_reached_downloader) + + def mark(self, request): + meta = _get_from_request(request, 'meta', None) + if not isinstance(meta, dict): + raise ValueError('No meta attribute in %s' % (request, )) + meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) + + def check_mark(self, request): + return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) + + def pop(self): + slots = [(d, s) for s,d in self._slots.items() if s in self.pqueues] + + if not slots: + return + + slot = min(slots)[1] + request, _ = self.pop_slot(slot) + self.mark(request) + return request + + def push(self, request, priority): + slot, _ = self.push_slot(request, priority) + if slot not in self._slots: + self._slots[slot] = 0 + + def on_response_download(self, response, request, spider): + if not self.check_mark(request): + return + + slot = _scheduler_slot_read(request) + if slot not in self._slots or self._slots[slot] <= 0: + raise ValueError('Get response for wrong slot "%s"' % (slot, )) + self._slots[slot] = self._slots[slot] - 1 + if self._slots[slot] == 0 and slot not in self.pqueues: + del self._slots[slot] + + def on_request_reached_downloader(self, request, spider): + if not self.check_mark(request): + return + + slot = _scheduler_slot_read(request) + self._slots[slot] = self._slots.get(slot, 0) + 1 + + def close(self): + self._slots.clear() + return super(DownloaderAwarePriorityQueue, self).close() diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 000000000..fd86e8d8c --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,315 @@ +import contextlib +import shutil +import tempfile +import unittest + +from scrapy.crawler import Crawler +from scrapy.core.scheduler import Scheduler +from scrapy.http import Request +from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write +from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.spiders import Spider + +class MockCrawler(Crawler): + def __init__(self, priority_queue_cls, jobdir): + + settings = dict(LOG_UNSERIALIZABLE_REQUESTS=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter') + super(MockCrawler, self).__init__(Spider, settings) + + +class SchedulerHandler: + priority_queue_cls = None + jobdir = None + + def create_scheduler(self): + self.mock_crawler = MockCrawler(self.priority_queue_cls, self.jobdir) + self.scheduler = Scheduler.from_crawler(self.mock_crawler) + self.spider = Spider(name='spider') + self.scheduler.open(self.spider) + + def close_scheduler(self): + self.scheduler.close('finished') + self.mock_crawler.stop() + + def setUp(self): + self.create_scheduler() + + def tearDown(self): + self.close_scheduler() + + +_PRIORITIES = [("http://foo.com/a", -2), + ("http://foo.com/d", 1), + ("http://foo.com/b", -1), + ("http://foo.com/c", 0), + ("http://foo.com/e", 2)] + + +_URLS = {"http://foo.com/a", "http://foo.com/b", "http://foo.com/c"} + + +class BaseSchedulerInMemoryTester(SchedulerHandler): + def test_length(self): + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(_URLS)) + + def test_dequeue(self): + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + urls = set() + while self.scheduler.has_pending_requests(): + urls.add(self.scheduler.next_request().url) + + self.assertEqual(urls, _URLS) + + def test_dequeue_priorities(self): + for url, priority in _PRIORITIES: + self.scheduler.enqueue_request(Request(url, priority=priority)) + + priorities = list() + while self.scheduler.has_pending_requests(): + priorities.append(self.scheduler.next_request().priority) + + self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + + +class BaseSchedulerOnDiskTester(SchedulerHandler): + + def setUp(self): + self.jobdir = tempfile.mkdtemp() + self.create_scheduler() + + def tearDown(self): + self.close_scheduler() + + shutil.rmtree(self.jobdir) + self.jobdir = None + + def test_length(self): + self.assertFalse(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), 0) + + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + self.close_scheduler() + self.create_scheduler() + + self.assertTrue(self.scheduler.has_pending_requests()) + self.assertEqual(len(self.scheduler), len(_URLS)) + + def test_dequeue(self): + for url in _URLS: + self.scheduler.enqueue_request(Request(url)) + + self.close_scheduler() + self.create_scheduler() + + urls = set() + while self.scheduler.has_pending_requests(): + urls.add(self.scheduler.next_request().url) + + self.assertEqual(urls, _URLS) + + def test_dequeue_priorities(self): + for url, priority in _PRIORITIES: + self.scheduler.enqueue_request(Request(url, priority=priority)) + + self.close_scheduler() + self.create_scheduler() + + priorities = list() + while self.scheduler.has_pending_requests(): + priorities.append(self.scheduler.next_request().priority) + + self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + + +class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): + priority_queue_cls = 'queuelib.PriorityQueue' + + +class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): + priority_queue_cls = 'queuelib.PriorityQueue' + + +_SLOTS = [("http://foo.com/a", 'a'), + ("http://foo.com/b", 'a'), + ("http://foo.com/c", 'b'), + ("http://foo.com/d", 'b'), + ("http://foo.com/e", 'c'), + ("http://foo.com/f", 'c')] + + +class TestSchedulerWithRoundRobinInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + + def test_round_robin(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + slots = list() + while self.scheduler.has_pending_requests(): + slots.append(_scheduler_slot_read(self.scheduler.next_request())) + + for i in range(0, len(_SLOTS), 2): + self.assertNotEqual(slots[i], slots[i+1]) + + def test_is_meta_set(self): + url = "http://foo.com/a" + request = Request(url) + if _scheduler_slot_read(request): + _scheduler_slot_write(request, None) + self.scheduler.enqueue_request(request) + self.assertIsNotNone(_scheduler_slot_read(request, None), None) + + +class TestSchedulerWithRoundRobinOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + + def test_round_robin(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + self.close_scheduler() + self.create_scheduler() + + slots = list() + while self.scheduler.has_pending_requests(): + slots.append(_scheduler_slot_read(self.scheduler.next_request())) + + for i in range(0, len(_SLOTS), 2): + self.assertNotEqual(slots[i], slots[i+1]) + + def test_is_meta_set(self): + url = "http://foo.com/a" + request = Request(url) + if _scheduler_slot_read(request): + _scheduler_slot_write(request, None) + self.scheduler.enqueue_request(request) + + self.close_scheduler() + self.create_scheduler() + + self.assertIsNotNone(_scheduler_slot_read(request, None), None) + + +@contextlib.contextmanager +def mkdtemp(): + dir = tempfile.mkdtemp() + try: + yield dir + finally: + shutil.rmtree(dir) + + +def _migration(): + + with mkdtemp() as tmp_dir: + prev_scheduler_handler = SchedulerHandler() + prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue' + prev_scheduler_handler.jobdir = tmp_dir + + prev_scheduler_handler.create_scheduler() + for url in _URLS: + prev_scheduler_handler.scheduler.enqueue_request(Request(url)) + prev_scheduler_handler.close_scheduler() + + next_scheduler_handler = SchedulerHandler() + next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + next_scheduler_handler.jobdir = tmp_dir + + next_scheduler_handler.create_scheduler() + + +class TestMigration(unittest.TestCase): + def test_migration(self): + self.assertRaises(ValueError, _migration) + + +class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + + def test_logic(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + slots = list() + requests = list() + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + slots.append(_scheduler_slot_read(request)) + self.mock_crawler.signals.send_catch_log( + signal=request_reached_downloader, + request=request, + spider=self.spider + ) + requests.append(request) + self.assertEqual(len(slots), len(_SLOTS)) + + for request in requests: + self.mock_crawler.signals.send_catch_log(signal=response_downloaded, + request=request, + response=None, + spider=self.spider) + + unique_slots = len(set(s for _, s in _SLOTS)) + for i in range(0, len(_SLOTS), unique_slots): + part = slots[i:i + unique_slots] + self.assertEqual(len(part), len(set(part))) + + +class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): + priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + def test_logic(self): + for url, slot in _SLOTS: + request = Request(url) + _scheduler_slot_write(request, slot) + self.scheduler.enqueue_request(request) + + self.close_scheduler() + self.create_scheduler() + + slots = list() + requests = list() + while self.scheduler.has_pending_requests(): + request = self.scheduler.next_request() + slots.append(_scheduler_slot_read(request)) + self.mock_crawler.signals.send_catch_log( + signal=request_reached_downloader, + request=request, + spider=self.spider + ) + requests.append(request) + + self.assertEqual(self.scheduler.mqs._slots, {}) + self.assertEqual(len(slots), len(_SLOTS)) + + for request in requests: + self.mock_crawler.signals.send_catch_log(signal=response_downloaded, + request=request, + response=None, + spider=self.spider) + + unique_slots = len(set(s for _, s in _SLOTS)) + for i in range(0, len(_SLOTS), unique_slots): + part = slots[i:i + unique_slots] + self.assertEqual(len(part), len(set(part))) From afdb69ea6daac8bd4f580d6c20bf9e93b741957b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 16:36:05 +0100 Subject: [PATCH 048/140] Add a troubleshooting section to the installation instructions Its initial content covers the workaround for #2473. --- docs/intro/install.rst | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 4a9aa3cfb..daec7fcb7 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -30,7 +30,8 @@ dependencies depending on your operating system, so be sure to check the We strongly recommend that you install Scrapy in :ref:`a dedicated virtualenv `, to avoid conflicting with your system packages. -For more detailed and platform specifics instructions, read on. +For more detailed and platform specifics instructions, as well as +troubleshooting information, read on. Things that are good to know @@ -247,6 +248,34 @@ that setuptools was unable to pick up one PyPy-specific dependency. To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +.. _intro-install-troubleshooting: + +Troubleshooting +=============== + +AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' +---------------------------------------------------------------- + +After you install or upgrade Scrapy, Twisted or pyOpenSSL, you may get an +exception with the following traceback:: + + […] + File "[…]/site-packages/twisted/protocols/tls.py", line 63, in + from twisted.internet._sslverify import _setAcceptableProtocols + File "[…]/site-packages/twisted/internet/_sslverify.py", line 38, in + TLSVersion.TLSv1_1: SSL.OP_NO_TLSv1_1, + AttributeError: 'module' object has no attribute 'OP_NO_TLSv1_1' + +The reason you get this exception is that your system or virtual environment +has a version of pyOpenSSL that your version of Twisted does not support. + +To install a version of pyOpenSSL that your version of Twisted supports, +reinstall Twisted with the :code:`tls` extra option:: + + pip install twisted[tls] + +For details, see `Issue #2473 `_. + .. _Python: https://www.python.org/ .. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: http://lxml.de/ From 9c314800e4b195df41e5c0aba0d9ffe4bcffec8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Dec 2018 17:14:10 +0100 Subject: [PATCH 049/140] Document the SCRAPY_PROJECT environment variable Fixes #1109 --- docs/topics/commands.rst | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index ef9c45196..97f8311de 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -37,7 +37,7 @@ Scrapy also understands, and can be configured through, a number of environment variables. Currently these are: * ``SCRAPY_SETTINGS_MODULE`` (see :ref:`topics-settings-module-envvar`) -* ``SCRAPY_PROJECT`` +* ``SCRAPY_PROJECT`` (see :ref:`topics-project-envvar`) * ``SCRAPY_PYTHON_SHELL`` (see :ref:`topics-shell`) .. _topics-project-structure: @@ -71,6 +71,33 @@ the project settings. Here is an example:: [settings] default = myproject.settings +.. _topics-project-envvar: + +Sharing the root directory between projects +=========================================== + +A project root directory, the one that contains the ``scrapy.cfg``, may be +shared by multiple Scrapy projects, each with its own settings module. + +In that case, you must define one or more aliases for those settings modules +under ``[settings]`` in your ``scrapy.cfg`` file:: + + [settings] + default = myproject1.settings + project1 = myproject1.settings + project2 = myproject2.settings + +By default, the ``scrapy`` command-line tool will use the ``default`` settings. +Use the ``SCRAPY_PROJECT`` environment variable to specify a different project +for ``scrapy`` to use:: + + $ scrapy settings --get BOT_NAME + Project 1 Bot + $ export SCRAPY_PROJECT=project2 + $ scrapy settings --get BOT_NAME + Project 2 Bot + + Using the ``scrapy`` tool ========================= From f56079f6c71a77c1f70510cf291cd808617933cd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 5 Dec 2018 10:02:42 +0000 Subject: [PATCH 050/140] Test cleanups PEP8 fixes no need to close implicitly do not use pytest need to put it into class remove round-robin queue additional check for empty queue use pytest tmpdir fixture --- scrapy/pqueues.py | 50 ++++------------ tests/test_scheduler.py | 128 ++++++++++++---------------------------- 2 files changed, 50 insertions(+), 128 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75073b7a4..287a8de35 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,4 +1,3 @@ -from collections import deque import hashlib import logging from six import text_type @@ -71,16 +70,17 @@ class PrioritySlot: self.slot = slot def __hash__(self): - return hash((self.priority, self.slot)) + return hash((self.priority, self.slot)) def __eq__(self, other): - return (self.priority, self.slot) == (other.priority, other.slot) + return (self.priority, self.slot) == (other.priority, other.slot) def __lt__(self, other): - return (self.priority, self.slot) < (other.priority, other.slot) + return (self.priority, self.slot) < (other.priority, other.slot) def __str__(self): - return '_'.join([text_type(self.priority), _pathable(text_type(self.slot))]) + return '_'.join([text_type(self.priority), + _pathable(text_type(self.slot))]) class PriorityAsTupleQueue(PriorityQueue): @@ -135,9 +135,10 @@ class SlotBasedPriorityQueue(object): slot = _scheduler_slot(request) is_new = False if slot not in self.pqueues: - is_new = True self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory) - self.pqueues[slot].push(request, PrioritySlot(priority=priority, slot=slot)) + queue = self.pqueues[slot] + is_new = queue.is_empty() + queue.push(request, PrioritySlot(priority=priority, slot=slot)) return slot, is_new def close(self): @@ -152,36 +153,6 @@ class SlotBasedPriorityQueue(object): return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 -class RoundRobinPriorityQueue(SlotBasedPriorityQueue): - - def __init__(self, qfactory, startprios={}): - super(RoundRobinPriorityQueue, self).__init__(qfactory, startprios) - self._slots = deque() - for slot in self.pqueues: - self._slots.append(slot) - - def push(self, request, priority): - slot, is_new = self.push_slot(request, priority) - if is_new: - self._slots.append(slot) - - def pop(self): - if not self._slots: - return - - slot = self._slots.popleft() - request, is_empty = self.pop_slot(slot) - - if not is_empty: - self._slots.append(slot) - - return request - - def close(self): - self._slots.clear() - return super(RoundRobinPriorityQueue, self).close() - - class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' @@ -191,7 +162,8 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return cls(crawler, qfactory, startprios) def __init__(self, crawler, qfactory, startprios={}): - super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, + startprios) self._slots = {slot: 0 for slot in self.pqueues} crawler.signals.connect(self.on_response_download, signal=response_downloaded) @@ -208,7 +180,7 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) def pop(self): - slots = [(d, s) for s,d in self._slots.items() if s in self.pqueues] + slots = [(d, s) for s, d in self._slots.items() if s in self.pqueues] if not slots: return diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index fd86e8d8c..e1cf5842d 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,4 +1,3 @@ -import contextlib import shutil import tempfile import unittest @@ -10,15 +9,18 @@ from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write from scrapy.signals import request_reached_downloader, response_downloaded from scrapy.spiders import Spider + class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): - settings = dict(LOG_UNSERIALIZABLE_REQUESTS=False, - SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', - SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', - SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, - JOBDIR=jobdir, - DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter') + settings = dict( + LOG_UNSERIALIZABLE_REQUESTS=False, + SCHEDULER_DISK_QUEUE='scrapy.squeues.PickleLifoDiskQueue', + SCHEDULER_MEMORY_QUEUE='scrapy.squeues.LifoMemoryQueue', + SCHEDULER_PRIORITY_QUEUE=priority_queue_cls, + JOBDIR=jobdir, + DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' + ) super(MockCrawler, self).__init__(Spider, settings) @@ -82,7 +84,8 @@ class BaseSchedulerInMemoryTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + self.assertEqual(priorities, + sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) class BaseSchedulerOnDiskTester(SchedulerHandler): @@ -134,7 +137,8 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): while self.scheduler.has_pending_requests(): priorities.append(self.scheduler.next_request().priority) - self.assertEqual(priorities, sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) + self.assertEqual(priorities, + sorted([x[1] for x in _PRIORITIES], key=lambda x: -x)) class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): @@ -153,75 +157,15 @@ _SLOTS = [("http://foo.com/a", 'a'), ("http://foo.com/f", 'c')] -class TestSchedulerWithRoundRobinInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): - priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' +class TestMigration(unittest.TestCase): - def test_round_robin(self): - for url, slot in _SLOTS: - request = Request(url) - _scheduler_slot_write(request, slot) - self.scheduler.enqueue_request(request) + def setUp(self): + self.tmpdir = tempfile.mkdtemp() - slots = list() - while self.scheduler.has_pending_requests(): - slots.append(_scheduler_slot_read(self.scheduler.next_request())) + def tearDown(self): + shutil.rmtree(self.tmpdir) - for i in range(0, len(_SLOTS), 2): - self.assertNotEqual(slots[i], slots[i+1]) - - def test_is_meta_set(self): - url = "http://foo.com/a" - request = Request(url) - if _scheduler_slot_read(request): - _scheduler_slot_write(request, None) - self.scheduler.enqueue_request(request) - self.assertIsNotNone(_scheduler_slot_read(request, None), None) - - -class TestSchedulerWithRoundRobinOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): - priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' - - def test_round_robin(self): - for url, slot in _SLOTS: - request = Request(url) - _scheduler_slot_write(request, slot) - self.scheduler.enqueue_request(request) - - self.close_scheduler() - self.create_scheduler() - - slots = list() - while self.scheduler.has_pending_requests(): - slots.append(_scheduler_slot_read(self.scheduler.next_request())) - - for i in range(0, len(_SLOTS), 2): - self.assertNotEqual(slots[i], slots[i+1]) - - def test_is_meta_set(self): - url = "http://foo.com/a" - request = Request(url) - if _scheduler_slot_read(request): - _scheduler_slot_write(request, None) - self.scheduler.enqueue_request(request) - - self.close_scheduler() - self.create_scheduler() - - self.assertIsNotNone(_scheduler_slot_read(request, None), None) - - -@contextlib.contextmanager -def mkdtemp(): - dir = tempfile.mkdtemp() - try: - yield dir - finally: - shutil.rmtree(dir) - - -def _migration(): - - with mkdtemp() as tmp_dir: + def _migration(self, tmp_dir): prev_scheduler_handler = SchedulerHandler() prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue' prev_scheduler_handler.jobdir = tmp_dir @@ -232,18 +176,18 @@ def _migration(): prev_scheduler_handler.close_scheduler() next_scheduler_handler = SchedulerHandler() - next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.RoundRobinPriorityQueue' + next_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' next_scheduler_handler.jobdir = tmp_dir next_scheduler_handler.create_scheduler() - -class TestMigration(unittest.TestCase): def test_migration(self): - self.assertRaises(ValueError, _migration) + with self.assertRaises(ValueError): + self._migration(self.tmpdir) -class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): +class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, + unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): @@ -266,10 +210,12 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unit self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log(signal=response_downloaded, - request=request, - response=None, - spider=self.spider) + self.mock_crawler.signals.send_catch_log( + signal=response_downloaded, + request=request, + response=None, + spider=self.spider + ) unique_slots = len(set(s for _, s in _SLOTS)) for i in range(0, len(_SLOTS), unique_slots): @@ -277,8 +223,10 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, unit self.assertEqual(len(part), len(set(part))) -class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): +class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, + unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + def test_logic(self): for url, slot in _SLOTS: request = Request(url) @@ -304,10 +252,12 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log(signal=response_downloaded, - request=request, - response=None, - spider=self.spider) + self.mock_crawler.signals.send_catch_log( + signal=response_downloaded, + request=request, + response=None, + spider=self.spider + ) unique_slots = len(set(s for _, s in _SLOTS)) for i in range(0, len(_SLOTS), unique_slots): From 7efba101946af93397ec3c2323b920644e20ce04 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Mon, 10 Dec 2018 14:44:15 +0800 Subject: [PATCH 051/140] remove "sudo: false" now that travis no longer supports it https://changelog.travis-ci.com/deprecation-container-based-linux-build-environment-82037 --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4218d13bf..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: python -sudo: false branches: only: - master From 0e06b9a81672ec432d2fccc3cbacc823ea47b656 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:35:18 +0000 Subject: [PATCH 052/140] use urlparse_cached where it is possible --- scrapy/pqueues.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 287a8de35..ff7ec8c8a 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -8,6 +8,7 @@ from queuelib import PriorityQueue from scrapy.core.downloader import Downloader from scrapy.http import Request from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) @@ -41,11 +42,26 @@ def _scheduler_slot_write(request, slot): def _scheduler_slot(request): - slot = _scheduler_slot_read(request, None) - if slot is None: - url = _get_from_request(request, 'url') + if isinstance(request, dict): + meta = request.get('meta', dict()) + elif isinstance(request, Request): + meta = request.meta + else: + raise ValueError('Bad type of request "%s"' % (request.__class__, )) + + slot = meta.get(SCHEDULER_SLOT_META_KEY, None) + + if slot is not None: + return slot + + if isinstance(request, dict): + url = request.get('url', None) slot = urlparse(url).hostname or '' - _scheduler_slot_write(request, slot) + elif isinstance(request, Request): + url = request.url + slot = urlparse_cached(request).hostname or '' + + meta[SCHEDULER_SLOT_META_KEY] = slot return slot From 484927b08caff66ea622f8553468c831154df30a Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:38:28 +0000 Subject: [PATCH 053/140] less complex implementation --- scrapy/pqueues.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index ff7ec8c8a..538678345 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -28,16 +28,11 @@ def _get_from_request(request, key, default=None): def _scheduler_slot_read(request, default=None): - meta = _get_from_request(request, 'meta', dict()) - slot = meta.get(SCHEDULER_SLOT_META_KEY, default) - return slot + return request.meta.get(SCHEDULER_SLOT_META_KEY, default) def _scheduler_slot_write(request, slot): - meta = _get_from_request(request, 'meta', None) - if not isinstance(meta, dict): - raise ValueError('No meta attribute in %s' % (request, )) - meta[SCHEDULER_SLOT_META_KEY] = slot + request.meta[SCHEDULER_SLOT_META_KEY] = slot def _scheduler_slot(request): From 6af964cc0b47c570e035a3486b9f8aebd349bd84 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:54:24 +0000 Subject: [PATCH 054/140] common indentation for comment --- scrapy/pqueues.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 538678345..31e90ff12 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -96,9 +96,9 @@ class PrioritySlot: class PriorityAsTupleQueue(PriorityQueue): """ - Python structures is not directly (de)serialized (to)from json. - We need this modified queue to transform custom structure (from)to - json serializable structures + Python structures is not directly (de)serialized (to)from json. + We need this modified queue to transform custom structure (from)to + json serializable structures """ def __init__(self, qfactory, startprios=()): From a46613afa8acd136f4ba62df2ced2f3c87679512 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 14:55:06 +0000 Subject: [PATCH 055/140] use regular comments --- scrapy/pqueues.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 31e90ff12..75fc198d0 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -64,10 +64,8 @@ def _scheduler_slot(request): def _pathable(x): pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x]) - """ - as we replace some letters we can get collision for different slots - add we add unique part - """ + # as we replace some letters we can get collision for different slots + # add we add unique part unique_slot = hashlib.md5(x.encode('utf8')).hexdigest() return '-'.join([pathable_slot, unique_slot]) From a23e1894b3a09e1daf49dd9592546b2d21bc9a72 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 14 Dec 2018 16:18:34 +0000 Subject: [PATCH 056/140] Fix boto problem another way to fix boto problem Revert "fix for travis ci based on https://github.com/boto/boto/issues/3717" This reverts commit 150d2564ff0ea994652da7f5be333d72e0b38d93. fix for travis ci based on https://github.com/boto/boto/issues/3717 --- tests/requirements-py2.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index 790f29d34..f5bcfda60 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -11,3 +11,4 @@ testfixtures # optional for shell wrapper tests bpython ipython<6.0 +google-compute-engine From d970be64cc47c382bd615cd547e7e94c17e27b48 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 17 Dec 2018 13:52:11 +0000 Subject: [PATCH 057/140] Integration test integration testing only everything is working, not logic of PQ use method create slot attribute in constructor corect class for test case stop crawler in teardown method use class correct entity naming python 2 adaptation integration test with crawler and spider --- tests/test_scheduler.py | 46 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index e1cf5842d..9bdc82b30 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,12 +2,17 @@ import shutil import tempfile import unittest +from twisted.internet import defer +from twisted.trial.unittest import TestCase + from scrapy.crawler import Crawler from scrapy.core.scheduler import Scheduler from scrapy.http import Request from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write from scrapy.signals import request_reached_downloader, response_downloaded from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer class MockCrawler(Crawler): @@ -223,6 +228,13 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, self.assertEqual(len(part), len(set(part))) +def _is_slots_unique(base_slots, result_slots): + unique_slots = len(set(s for _, s in base_slots)) + for i in range(0, len(result_slots), unique_slots): + part = result_slots[i:i + unique_slots] + assert len(part) == len(set(part)) + + class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' @@ -259,7 +271,33 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, spider=self.spider ) - unique_slots = len(set(s for _, s in _SLOTS)) - for i in range(0, len(_SLOTS), unique_slots): - part = slots[i:i + unique_slots] - self.assertEqual(len(part), len(set(part))) + _is_slots_unique(_SLOTS, slots) + + +class StartUrlsSpider(Spider): + + def __init__(self, start_urls): + self.start_urls = start_urls + + +class TestIntegrationWithDownloaderAwareOnDisk(TestCase): + def setUp(self): + self.crawler = get_crawler( + StartUrlsSpider, + {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue', + 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'} + ) + + @defer.inlineCallbacks + def tearDown(self): + yield self.crawler.stop() + + @defer.inlineCallbacks + def test_integration_downloader_aware_priority_queue(self): + with MockServer() as mockserver: + + url = mockserver.url("/status?n=200", is_secure=False) + slots = [url] * 6 + yield self.crawler.crawl(slots) + self.assertEqual(self.crawler.stats.get_value('downloader/response_count'), + len(slots)) From 7d3175ac8433f964ebbb80ebd67f9899cf059100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:23:23 -0300 Subject: [PATCH 058/140] Fix boto import error under Jessie testing environment --- .travis.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 08b0bf119..a201f97b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,11 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi + if [ "$TOXENV" = "jessie" ]; then + # Not used directly but allows boto GCE plugins to load. + # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + pip install google-compute-engine + fi - pip install -U tox twine wheel codecov script: tox From 6ff2574c277ba1eda31fb43f86f43d5b7b4bef09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Thu, 20 Dec 2018 19:39:29 -0300 Subject: [PATCH 059/140] Needs to be installed within tox env --- .travis.yml | 5 ----- tox.ini | 3 +++ 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index a201f97b1..08b0bf119 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,11 +42,6 @@ install: virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION" source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate" fi - if [ "$TOXENV" = "jessie" ]; then - # Not used directly but allows boto GCE plugins to load. - # https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 - pip install google-compute-engine - fi - pip install -U tox twine wheel codecov script: tox diff --git a/tox.ini b/tox.ini index e5543fe2a..0c0f8f7b7 100644 --- a/tox.ini +++ b/tox.ini @@ -51,6 +51,9 @@ deps = cssselect==0.9.1 zope.interface==4.1.1 -rtests/requirements-py2.txt +# Not used directly but allows boto GCE plugins to load. +# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262 + google-compute-engine==2.8.12 [testenv:trunk] basepython = python2.7 From 4163a7a1c7ac11c8d4db70f371c26181b90d8dfd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 21 Dec 2018 09:10:32 +0000 Subject: [PATCH 060/140] no need for this --- tests/requirements-py2.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index f5bcfda60..790f29d34 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -11,4 +11,3 @@ testfixtures # optional for shell wrapper tests bpython ipython<6.0 -google-compute-engine From 987c2ae4a964e45120c245235c9b0c49dc36b71f Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:13:09 +0000 Subject: [PATCH 061/140] test ip concurrency incompatibility with DAPQ --- tests/test_scheduler.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 9bdc82b30..17b706bd7 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -301,3 +301,20 @@ class TestIntegrationWithDownloaderAwareOnDisk(TestCase): yield self.crawler.crawl(slots) self.assertEqual(self.crawler.stats.get_value('downloader/response_count'), len(slots)) + + +class TestIncompatibility(unittest.TestCase): + + def _incompatible(self): + settings = dict( + SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue', + CONCURRENT_REQUESTS_PER_IP=1 + ) + crawler = Crawler(Spider, settings) + scheduler = Scheduler.from_crawler(crawler) + spider = Spider(name='spider') + scheduler.open(spider) + + def test_incompatibility(self): + with self.assertRaises(ValueError): + self._incompatible() From 8e8ce301b1a56e40f7e9c322a7b73b8dcfcefc43 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:14:09 +0000 Subject: [PATCH 062/140] check CONCURRENT_REQUESTS_PER_IP is not set --- scrapy/pqueues.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 75fc198d0..d9effc9d1 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -171,6 +171,14 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return cls(crawler, qfactory, startprios) def __init__(self, crawler, qfactory, startprios={}): + ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' + ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) + + if ip_concurrency > 0: + raise ValueError('"%s" does not support %s=%d' % (self.__class__, + ip_concurrency_key, + ip_concurrency)) + super(DownloaderAwarePriorityQueue, self).__init__(qfactory, startprios) self._slots = {slot: 0 for slot in self.pqueues} From 338b78d796de6c93af0f4bcb762f82f5a14b87cd Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 25 Dec 2018 09:44:20 +0000 Subject: [PATCH 063/140] Add documentation add section to broad-crawl topic reword in accord with broad-crawl topic add documentation for new priority queue --- docs/topics/broad-crawls.rst | 11 +++++++++++ docs/topics/settings.rst | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index eb02086dc..37f7a8748 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -39,6 +39,17 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. +Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` +============================================== + +Default scrapy's scheduler priority queue is ``'queuelib.PriorityQueue'``. +It works best during single domain crawl. And it does not work well with crawling +many different domains in parallel + +To apply recommended priority queue use:: + + SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + Increase concurrency ==================== diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 47b6cf13d..7b9ff7e39 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1144,7 +1144,12 @@ SCHEDULER_PRIORITY_QUEUE ------------------------ Default: ``'queuelib.PriorityQueue'`` -Type of priority queue used by scheduler. +Type of priority queue used by scheduler. Another available type is +``scrapy.pqueues.DownloaderAwarePriorityQueue``. +``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than +``'queuelib.PriorityQueue'`` when you crawl many different domains in parallel. +But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. .. setting:: SPIDER_CONTRACTS From bbf24b7a1ce2e91eab57d1b8524d398822a1ddd1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 18:02:31 -0300 Subject: [PATCH 064/140] Rule.process_request: use scrapy.utils.python.get_func_args --- scrapy/spiders/crawl.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index c01f75798..f474b0a18 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -10,6 +10,7 @@ import six from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output +from scrapy.utils.python import get_func_args from scrapy.spiders import Spider @@ -35,10 +36,8 @@ class Rule(object): Wrapper around the request processing function to maintain backward compatibility with functions that do not take a Response object as parameter. """ - argcount = self.process_request.__code__.co_argcount - if hasattr(self.process_request, '__self__'): - argcount = argcount - 1 - args = [request] if argcount == 1 else [request, response] + arg_count = len(get_func_args(self.process_request)) + args = [request] if arg_count == 1 else [request, response] return self.process_request(*args) From 56929e77d98391255b77ffd3350abb49da18009e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 18:34:55 -0300 Subject: [PATCH 065/140] Rule.process_request: deprecate the use of functions taking only one argument --- scrapy/spiders/crawl.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f474b0a18..f469891d0 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -6,16 +6,19 @@ See documentation in docs/topics/spiders.rst """ import copy +import warnings + import six +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, HtmlResponse from scrapy.utils.spider import iterate_spider_output from scrapy.utils.python import get_func_args from scrapy.spiders import Spider -def identity(x): - return x +def _identity(request, response): + return request class Rule(object): @@ -25,19 +28,21 @@ class Rule(object): self.callback = callback self.cb_kwargs = cb_kwargs or {} self.process_links = process_links - self.process_request = process_request or identity - if follow is None: - self.follow = False if callback else True - else: - self.follow = follow + self.process_request = process_request or _identity + self.follow = follow if follow is not None else not callback def _process_request(self, request, response): """ - Wrapper around the request processing function to maintain backward compatibility - with functions that do not take a Response object as parameter. + Wrapper around the request processing function to maintain backward + compatibility with functions that do not take a Response object """ arg_count = len(get_func_args(self.process_request)) - args = [request] if arg_count == 1 else [request, response] + if arg_count == 1: + args = [request] + msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' + warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + else: + args = [request, response] return self.process_request(*args) From 174ba3cc5671cdc9e66cb29275986ff7481affc5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 19:16:18 -0300 Subject: [PATCH 066/140] Rule.process_request: update docs --- docs/topics/spiders.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 24b6f7ec9..30e15906e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -403,11 +403,11 @@ Crawling rules This is mainly used for filtering purposes. ``process_request`` is a callable (or a string, in which case a method from - the spider object with that name will be used) which will be called for - every request extracted by this rule. This callable should take a Request object - as first positional argument and, optionally, the Response object from which the - Request originated as second positional argument. It must return a request or None - (to filter out the request). + the spider object with that name will be used) which will be called for every + :class:`~scrapy.http.Request` extracted by this rule. This callable should + take said request as first argument and the :class:`~scrapy.http.Response` + from which the request originated as second argument. It must return a + ``Request`` object or ``None`` (to filter out the request). CrawlSpider example ~~~~~~~~~~~~~~~~~~~ From 1b4385b7e3f78694c0378455644b539d80d293a2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 22 Mar 2019 19:46:17 -0300 Subject: [PATCH 067/140] Rule.process_request: move deprecation warnings and compiling code, update tests --- scrapy/spiders/crawl.py | 35 +++++++++++++++++++---------------- tests/test_spider.py | 38 ++++++++++++++++++++++---------------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f469891d0..6db3a1e06 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -21,6 +21,13 @@ def _identity(request, response): return request +def _get_method(method, spider): + if callable(method): + return method + elif isinstance(method, six.string_types): + return getattr(spider, method, None) + + class Rule(object): def __init__(self, link_extractor, callback=None, cb_kwargs=None, follow=None, process_links=None, process_request=None): @@ -29,20 +36,24 @@ class Rule(object): self.cb_kwargs = cb_kwargs or {} self.process_links = process_links self.process_request = process_request or _identity + self.process_request_argcount = None self.follow = follow if follow is not None else not callback + def _compile(self, spider): + self.callback = _get_method(self.callback, spider) + self.process_links = _get_method(self.process_links, spider) + self.process_request = _get_method(self.process_request, spider) + self.process_request_argcount = len(get_func_args(self.process_request)) + if self.process_request_argcount == 1: + msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' + warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) + def _process_request(self, request, response): """ Wrapper around the request processing function to maintain backward compatibility with functions that do not take a Response object """ - arg_count = len(get_func_args(self.process_request)) - if arg_count == 1: - args = [request] - msg = 'Rule.process_request should accept two arguments (request, response), accepting only one is deprecated' - warnings.warn(msg, category=ScrapyDeprecationWarning, stacklevel=2) - else: - args = [request, response] + args = [request] if self.process_request_argcount == 1 else [request, response] return self.process_request(*args) @@ -98,17 +109,9 @@ class CrawlSpider(Spider): yield request_or_item def _compile_rules(self): - def get_method(method): - if callable(method): - return method - elif isinstance(method, six.string_types): - return getattr(self, method, None) - self._rules = [copy.copy(r) for r in self.rules] for rule in self._rules: - rule.callback = get_method(rule.callback) - rule.process_links = get_method(rule.process_links) - rule.process_request = get_method(rule.process_request) + rule._compile(self) @classmethod def from_crawler(cls, crawler, *args, **kwargs): diff --git a/tests/test_spider.py b/tests/test_spider.py index c9af7a2d7..83fb68c2f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -275,14 +275,17 @@ class CrawlSpiderTest(SpiderTest): Rule(LinkExtractor(), process_request=process_request_change_domain), ) - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://example.com/somepage/item/12.html', - 'http://example.com/about.html', - 'http://example.com/nofollow.html']) + with warnings.catch_warnings(record=True) as cw: + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://example.com/somepage/item/12.html', + 'http://example.com/about.html', + 'http://example.com/nofollow.html']) + self.assertEqual(len(cw), 1) + self.assertEqual(cw[0].category, ScrapyDeprecationWarning) def test_process_request_with_response(self): @@ -324,14 +327,17 @@ class CrawlSpiderTest(SpiderTest): def process_request_upper(self, request): return request.replace(url=request.url.upper()) - spider = _CrawlSpider() - output = list(spider._requests_to_follow(response)) - self.assertEqual(len(output), 3) - self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) - self.assertEqual([r.url for r in output], - ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', - 'http://EXAMPLE.ORG/ABOUT.HTML', - 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + with warnings.catch_warnings(record=True) as cw: + spider = _CrawlSpider() + output = list(spider._requests_to_follow(response)) + self.assertEqual(len(output), 3) + self.assertTrue(all(map(lambda r: isinstance(r, Request), output))) + self.assertEqual([r.url for r in output], + ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML', + 'http://EXAMPLE.ORG/ABOUT.HTML', + 'http://EXAMPLE.ORG/NOFOLLOW.HTML']) + self.assertEqual(len(cw), 1) + self.assertEqual(cw[0].category, ScrapyDeprecationWarning) def test_process_request_instance_method_with_response(self): From 90934959d07db881aec5933fd7b77bcd2dccfa4f Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 27 Dec 2018 17:12:24 +0500 Subject: [PATCH 068/140] actually apply __slots__ suggestion [wip] refactoring * SlotPriorityQueues doesn't care about objects inside, it is now just a container for multiple priority queues * assorted variable renames * don't inherit DownloaderAwarePriorityQueue from SlotBasedPriorityQueue * apply @whalebot-helmsman's suggestions for __slots__ and meta issues more bike-shedding * remove mutable default arguments * more verbose variable names remove unneeded code * PriorityAsTupleQueue.is_empty does the same as len(self) == 0 * custom PriorityAsTupleQueue.close is not needed after a switch to namedtuples * is_new and is_empty return values are unused * "url" local variable is unused PrioritySlot.__str__ shouldn't return unicode in Python 2 also, do some bike-shedding: _pathable -> _path_safe use namedtuple for PrioritySlot cleanup: _get_from_request does the same here Request.meta is always a dict --- scrapy/pqueues.py | 180 ++++++++++++++++++---------------------- tests/test_scheduler.py | 2 +- 2 files changed, 82 insertions(+), 100 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index d9effc9d1..3ef896b99 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,6 +1,6 @@ import hashlib import logging -from six import text_type +from collections import namedtuple from six.moves.urllib.parse import urlparse from queuelib import PriorityQueue @@ -17,12 +17,12 @@ logger = logging.getLogger(__name__) SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT -def _get_from_request(request, key, default=None): +def _get_request_meta(request): if isinstance(request, dict): - return request.get(key, default) + return request.setdefault('meta', {}) if isinstance(request, Request): - return getattr(request, key, default) + return request.meta raise ValueError('Bad type of request "%s"' % (request.__class__, )) @@ -35,15 +35,8 @@ def _scheduler_slot_write(request, slot): request.meta[SCHEDULER_SLOT_META_KEY] = slot -def _scheduler_slot(request): - - if isinstance(request, dict): - meta = request.get('meta', dict()) - elif isinstance(request, Request): - meta = request.meta - else: - raise ValueError('Bad type of request "%s"' % (request.__class__, )) - +def _set_scheduler_slot(request): + meta = _get_request_meta(request) slot = meta.get(SCHEDULER_SLOT_META_KEY, None) if slot is not None: @@ -53,43 +46,29 @@ def _scheduler_slot(request): url = request.get('url', None) slot = urlparse(url).hostname or '' elif isinstance(request, Request): - url = request.url slot = urlparse_cached(request).hostname or '' meta[SCHEDULER_SLOT_META_KEY] = slot - return slot -def _pathable(x): - pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in x]) - +def _path_safe(text): + """ Return a filesystem-safe version of a string ``text`` """ + pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' + for c in text]) # as we replace some letters we can get collision for different slots # add we add unique part - unique_slot = hashlib.md5(x.encode('utf8')).hexdigest() - + unique_slot = hashlib.md5(text.encode('utf8')).hexdigest() return '-'.join([pathable_slot, unique_slot]) -class PrioritySlot: - __slots__ = ('priority', 'slot') - - def __init__(self, priority=0, slot=None): - self.priority = priority - self.slot = slot - - def __hash__(self): - return hash((self.priority, self.slot)) - - def __eq__(self, other): - return (self.priority, self.slot) == (other.priority, other.slot) - - def __lt__(self, other): - return (self.priority, self.slot) < (other.priority, other.slot) +class PrioritySlot(namedtuple("PrioritySlot", ["priority", "slot"])): + """ ``(priority, slot)`` tuple which uses a path-safe slot name + when converting to str """ + __slots__ = () def __str__(self): - return '_'.join([text_type(self.priority), - _pathable(text_type(self.slot))]) + return '%s_%s' % (self.priority, _path_safe(str(self.slot))) class PriorityAsTupleQueue(PriorityQueue): @@ -99,78 +78,65 @@ class PriorityAsTupleQueue(PriorityQueue): json serializable structures """ def __init__(self, qfactory, startprios=()): - + startprios = [PrioritySlot(priority=p[0], slot=p[1]) + for p in startprios] super(PriorityAsTupleQueue, self).__init__( - qfactory, - [PrioritySlot(priority=p[0], slot=p[1]) for p in startprios] - ) - - def close(self): - startprios = super(PriorityAsTupleQueue, self).close() - return [(s.priority, s.slot) for s in startprios] - - def is_empty(self): - return not self.queues or len(self) == 0 + qfactory=qfactory, + startprios=startprios) -class SlotBasedPriorityQueue(object): +class SlotPriorityQueues(object): + """ Container for multiple priority queues. """ + def __init__(self, pqfactory, slot_startprios=None): + """ + ``pqfactory`` is a factory for creating new PriorityQueues. + It must be a function which accepts a single optional ``startprios`` + argument, with a list of priorities to create queues for. - def __init__(self, qfactory, startprios={}): - self.pqueues = dict() # slot -> priority queue - self.qfactory = qfactory # factory for creating new internal queues - - if not startprios: - return - - if not isinstance(startprios, dict): - raise ValueError("Looks like your priorities file malforfemed. " - "Possible reason: You run scrapy with previous " - "version. Interrupted it. Updated scrapy. And " - "run again.") - - for slot, prios in startprios.items(): - self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory, prios) + ``slot_startprios`` is a ``{slot: startprios}`` dict. + """ + self.pqfactory = pqfactory + self.pqueues = {} # slot -> priority queue + for slot, startprios in (slot_startprios or {}).items(): + self.pqueues[slot] = self.pqfactory(startprios) def pop_slot(self, slot): + """ Pop an object from a priority queue for this slot """ queue = self.pqueues[slot] request = queue.pop() - is_empty = queue.is_empty() - if is_empty: + if len(queue) == 0: del self.pqueues[slot] + return request - return request, is_empty - - def push_slot(self, request, priority): - slot = _scheduler_slot(request) - is_new = False + def push_slot(self, slot, obj, priority): + """ Push an object to a priority queue for this slot """ if slot not in self.pqueues: - self.pqueues[slot] = PriorityAsTupleQueue(self.qfactory) + self.pqueues[slot] = self.pqfactory() queue = self.pqueues[slot] - is_new = queue.is_empty() - queue.push(request, PrioritySlot(priority=priority, slot=slot)) - return slot, is_new + queue.push(obj, priority) def close(self): - startprios = dict() - for slot, queue in self.pqueues.items(): - prios = queue.close() - startprios[slot] = prios + active = {slot: queue.close() + for slot, queue in self.pqueues.items()} self.pqueues.clear() - return startprios + return active def __len__(self): return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0 + def __contains__(self, slot): + return slot in self.pqueues -class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): + +class DownloaderAwarePriorityQueue(object): _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' @classmethod - def from_crawler(cls, crawler, qfactory, startprios={}): + def from_crawler(cls, crawler, qfactory, startprios=None): return cls(crawler, qfactory, startprios) - def __init__(self, crawler, qfactory, startprios={}): + def __init__(self, crawler, qfactory, startprios=None): ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) @@ -179,16 +145,25 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): ip_concurrency_key, ip_concurrency)) - super(DownloaderAwarePriorityQueue, self).__init__(qfactory, - startprios) - self._slots = {slot: 0 for slot in self.pqueues} + def pqfactory(startprios=()): + return PriorityAsTupleQueue(qfactory, startprios) + + if startprios and not isinstance(startprios, dict): + raise ValueError("DownloaderAwarePriorityQueue accepts " + "``startprios`` as a dict; %r instance is passed." + " Only a crawl started with the same priority " + "queue class can be resumed." % startprios.__class__) + self._slot_pqueues = SlotPriorityQueues(pqfactory, + slot_startprios=startprios) + + self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, signal=response_downloaded) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) def mark(self, request): - meta = _get_from_request(request, 'meta', None) + meta = _get_request_meta(request) if not isinstance(meta, dict): raise ValueError('No meta attribute in %s' % (request, )) meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) @@ -197,39 +172,46 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) def pop(self): - slots = [(d, s) for s, d in self._slots.items() if s in self.pqueues] + slots = [(active_downloads, slot) + for slot, active_downloads in self._active_downloads.items() + if slot in self._slot_pqueues] if not slots: return slot = min(slots)[1] - request, _ = self.pop_slot(slot) + request = self._slot_pqueues.pop_slot(slot) self.mark(request) return request def push(self, request, priority): - slot, _ = self.push_slot(request, priority) - if slot not in self._slots: - self._slots[slot] = 0 + slot = _set_scheduler_slot(request) + priority_slot = PrioritySlot(priority=priority, slot=slot) + self._slot_pqueues.push_slot(slot, request, priority_slot) + if slot not in self._active_downloads: + self._active_downloads[slot] = 0 def on_response_download(self, response, request, spider): if not self.check_mark(request): return slot = _scheduler_slot_read(request) - if slot not in self._slots or self._slots[slot] <= 0: + if slot not in self._active_downloads or self._active_downloads[slot] <= 0: raise ValueError('Get response for wrong slot "%s"' % (slot, )) - self._slots[slot] = self._slots[slot] - 1 - if self._slots[slot] == 0 and slot not in self.pqueues: - del self._slots[slot] + self._active_downloads[slot] = self._active_downloads[slot] - 1 + if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues: + del self._active_downloads[slot] def on_request_reached_downloader(self, request, spider): if not self.check_mark(request): return slot = _scheduler_slot_read(request) - self._slots[slot] = self._slots.get(slot, 0) + 1 + self._active_downloads[slot] = self._active_downloads.get(slot, 0) + 1 def close(self): - self._slots.clear() - return super(DownloaderAwarePriorityQueue, self).close() + self._active_downloads.clear() + return self._slot_pqueues.close() + + def __len__(self): + return len(self._slot_pqueues) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 17b706bd7..5dd35f45c 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -260,7 +260,7 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, ) requests.append(request) - self.assertEqual(self.scheduler.mqs._slots, {}) + self.assertEqual(self.scheduler.mqs._active_downloads, {}) self.assertEqual(len(slots), len(_SLOTS)) for request in requests: From 757f53a32461ef0c3d2fe4caf64197f67271b5f3 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 9 Jan 2019 10:00:13 +0000 Subject: [PATCH 069/140] Address Lucy's comments add tests to check correctness of slot setermination unmark requests after downloading shorter better exception message --- scrapy/pqueues.py | 15 ++++++++++++--- tests/test_scheduler.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 3ef896b99..d8eed010f 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -36,6 +36,12 @@ def _scheduler_slot_write(request, slot): def _set_scheduler_slot(request): + """ + >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://bar.com'}) + False + >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://foo.com'}) + True + """ meta = _get_request_meta(request) slot = meta.get(SCHEDULER_SLOT_META_KEY, None) @@ -141,9 +147,8 @@ class DownloaderAwarePriorityQueue(object): ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) if ip_concurrency > 0: - raise ValueError('"%s" does not support %s=%d' % (self.__class__, - ip_concurrency_key, - ip_concurrency)) + raise ValueError('"%s" does not support setting %s' % (self.__class__, + ip_concurrency_key)) def pqfactory(startprios=()): return PriorityAsTupleQueue(qfactory, startprios) @@ -171,6 +176,9 @@ class DownloaderAwarePriorityQueue(object): def check_mark(self, request): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) + def unmark(self, request): + del request.meta[self._DOWNLOADER_AWARE_PQ_ID] + def pop(self): slots = [(active_downloads, slot) for slot, active_downloads in self._active_downloads.items() @@ -194,6 +202,7 @@ class DownloaderAwarePriorityQueue(object): def on_response_download(self, response, request, spider): if not self.check_mark(request): return + self.unmark(request) slot = _scheduler_slot_read(request) if slot not in self._active_downloads or self._active_downloads[slot] <= 0: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 5dd35f45c..3fb70a110 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -248,8 +248,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, self.close_scheduler() self.create_scheduler() - slots = list() - requests = list() + slots = [] + requests = [] while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() slots.append(_scheduler_slot_read(request)) From 3b1db71dac8716878ff1b94ee0d1095e5c80795f Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Wed, 9 Jan 2019 12:14:40 +0000 Subject: [PATCH 070/140] New signal update signature documentation for new signal utilize new signal correct signal handler signature emit new signal test another signal new signal rename test file faster test rename test case tests for signal emitting in bad cases --- docs/topics/signals.rst | 17 +++++++++ scrapy/core/downloader/__init__.py | 3 ++ scrapy/pqueues.py | 6 +-- scrapy/signals.py | 1 + tests/test_request_left.py | 59 ++++++++++++++++++++++++++++++ tests/test_scheduler.py | 8 ++-- 6 files changed, 86 insertions(+), 8 deletions(-) create mode 100644 tests/test_request_left.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index ff07b9d55..f13e8270c 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,6 +295,23 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object +request_left_downloader +--------------------------- + +.. signal:: request_left_downloader +.. function:: request_left_downloader(request, spider) + + Sent when a :class:`~scrapy.http.Request` left downloader even in case of + failure. + + The signal does not support returning deferreds from their handlers. + + :param request: the request that reached downloader + :type request: :class:`~scrapy.http.Request` object + + :param spider: the spider that yielded the request + :type spider: :class:`~scrapy.spiders.Spider` object + response_received ----------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 4695d75f4..d856a2f37 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -188,6 +188,9 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) + self.signals.send_catch_log(signal=signals.request_left_downloader, + request=request, + spider=spider) return _ return dfd.addBoth(finish_transferring) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index d8eed010f..6a9feb599 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -7,7 +7,7 @@ from queuelib import PriorityQueue from scrapy.core.downloader import Downloader from scrapy.http import Request -from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.utils.httpobj import urlparse_cached @@ -163,7 +163,7 @@ class DownloaderAwarePriorityQueue(object): self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, - signal=response_downloaded) + signal=request_left_downloader) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) @@ -199,7 +199,7 @@ class DownloaderAwarePriorityQueue(object): if slot not in self._active_downloads: self._active_downloads[slot] = 0 - def on_response_download(self, response, request, spider): + def on_response_download(self, request, spider): if not self.check_mark(request): return self.unmark(request) diff --git a/scrapy/signals.py b/scrapy/signals.py index c0e4bb74e..2ea986b8c 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,6 +14,7 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() +request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() diff --git a/tests/test_request_left.py b/tests/test_request_left.py new file mode 100644 index 000000000..ddeca0499 --- /dev/null +++ b/tests/test_request_left.py @@ -0,0 +1,59 @@ +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.signals import request_left_downloader +from scrapy.spiders import Spider +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer + +class SignalCatcherSpider(Spider): + name = 'signal_catcher' + + def __init__(self, crawler, url, *args, **kwargs): + super(SignalCatcherSpider, self).__init__(*args, **kwargs) + crawler.signals.connect(self.on_response_download, + signal=request_left_downloader) + self.catched_times = 0 + self.start_urls = [url] + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + spider = cls(crawler, *args, **kwargs) + return spider + + def on_response_download(self, request, spider): + self.catched_times = self.catched_times + 1 + + +class TestCatching(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + + @defer.inlineCallbacks + def test_success(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/status?n=200")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_timeout(self): + crawler = get_crawler(SignalCatcherSpider, + {'DOWNLOAD_TIMEOUT': 0.1}) + yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_disconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl(self.mockserver.url("/drop")) + self.assertEqual(crawler.spider.catched_times, 1) + + @defer.inlineCallbacks + def test_noconnect(self): + crawler = get_crawler(SignalCatcherSpider) + yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') + self.assertEqual(crawler.spider.catched_times, 1) diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 3fb70a110..1bcc1e5a8 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -9,7 +9,7 @@ from scrapy.crawler import Crawler from scrapy.core.scheduler import Scheduler from scrapy.http import Request from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write -from scrapy.signals import request_reached_downloader, response_downloaded +from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver import MockServer @@ -216,9 +216,8 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, for request in requests: self.mock_crawler.signals.send_catch_log( - signal=response_downloaded, + signal=request_left_downloader, request=request, - response=None, spider=self.spider ) @@ -265,9 +264,8 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, for request in requests: self.mock_crawler.signals.send_catch_log( - signal=response_downloaded, + signal=request_left_downloader, request=request, - response=None, spider=self.spider ) From 83eb5376458ce1d444e8ad7911730ee0c58c8544 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Thu, 17 Jan 2019 07:38:15 +0500 Subject: [PATCH 071/140] assorted cleanups: comments, docstrings, etc scheduler cleanup Scheduler no longer converts requests to dicts; PriorityQueue instances always work with Request instances; converting Requests to dicts is now Priority Queue responsibility. minor cleanup --- docs/topics/settings.rst | 6 +- scrapy/core/scheduler.py | 99 +++++++++++++---- scrapy/pqueues.py | 158 +++++++++++++++------------- scrapy/settings/default_settings.py | 2 +- scrapy/squeues.py | 11 +- 5 files changed, 175 insertions(+), 101 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 7b9ff7e39..6e13e64d6 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1142,13 +1142,13 @@ Type of in-memory queue used by scheduler. Other available type is: SCHEDULER_PRIORITY_QUEUE ------------------------ -Default: ``'queuelib.PriorityQueue'`` +Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. ``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than -``'queuelib.PriorityQueue'`` when you crawl many different domains in parallel. -But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different +domains in parallel. But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. .. setting:: SPIDER_CONTRACTS diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d40f3aa0c..c385fafe1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -1,17 +1,44 @@ import os import json import logging +import warnings from os.path import join, exists -from scrapy.utils.reqser import request_to_dict, request_from_dict +from queuelib import PriorityQueue + from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir +from scrapy.utils.deprecate import ScrapyDeprecationWarning + logger = logging.getLogger(__name__) class Scheduler(object): + """ + Scrapy Scheduler. It allows to enqueue requests and then get + a next request to download. Scheduler is also handling duplication + filtering, via dupefilter. + Prioritization and queueing is not performed by the Scheduler. + User sets ``priority`` field for each Request, and a PriorityQueue + (defined by :setting:`SCHEDULER_PRIORITY_QUEUE`) uses these priorities + to dequeue requests in a desired order. + + Scheduler uses two PriorityQueue instances, configured to work in-memory + and on-disk (optional). When on-disk queue is present, it is used by + default, and an in-memory queue is used as a fallback for cases where + a disk queue can't handle a request (can't serialize it). + + :setting:`SCHEDULER_MEMORY_QUEUE` and + :setting:`SCHEDULER_DISK_QUEUE` allow to specify lower-level queue classes + which PriorityQueue instances would be instantiated with, to keep requests + on disk and in memory respectively. + + Overall, Scheduler is an object which holds several PriorityQueue instances + (in-memory and on-disk) and implements fallback logic for them. + Also, it handles dupefilters. + """ def __init__(self, dupefilter, jobdir=None, dqclass=None, mqclass=None, logunser=False, stats=None, pqclass=None, crawler=None): self.df = dupefilter @@ -29,9 +56,19 @@ class Scheduler(object): dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) + if pqclass is PriorityQueue: + # backwards compatibility + warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" + " is no longer supported because of API changes; " + "please use 'scrapy.pqueues.ScrapyPriorityQueue'", + ScrapyDeprecationWarning) + from scrapy.pqueues import ScrapyPriorityQueue + pqclass = ScrapyPriorityQueue + dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) - logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', settings.getbool('SCHEDULER_DEBUG')) + logunser = settings.getbool('LOG_UNSERIALIZABLE_REQUESTS', + settings.getbool('SCHEDULER_DEBUG')) return cls(dupefilter, jobdir=job_dir(settings), logunser=logunser, stats=crawler.stats, pqclass=pqclass, dqclass=dqclass, mqclass=mqclass, crawler=crawler) @@ -41,15 +78,19 @@ class Scheduler(object): def open(self, spider): self.spider = spider - self.mqs = create_instance(self.pqclass, None, self.crawler, self._newmq) + + # in-memory PriorityQueue instance + self.mqs = self._mq() + + # on-disk PriorityQueue instance self.dqs = self._dq() if self.dqdir else None + return self.df.open() def close(self, reason): if self.dqs: - prios = self.dqs.close() - with open(join(self.dqdir, 'active.json'), 'w') as f: - json.dump(prios, f) + state = self.dqs.close() + self._write_dqs_state(self.dqdir, state) return self.df.close(reason) def enqueue_request(self, request): @@ -66,7 +107,7 @@ class Scheduler(object): return True def next_request(self): - request = self.mqs.pop() + request = self._mqpop() if request: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: @@ -84,8 +125,7 @@ class Scheduler(object): if self.dqs is None: return try: - reqd = request_to_dict(request, self.spider) - self.dqs.push(reqd, -request.priority) + self.dqs.push(request, -request.priority) except ValueError as e: # non serializable request if self.logunser: msg = ("Unable to serialize request: %(request)s - reason:" @@ -105,37 +145,54 @@ class Scheduler(object): def _dqpop(self): if self.dqs: - d = self.dqs.pop() - if d: - return request_from_dict(d, self.spider) + return self.dqs.pop() + + def _mqpop(self): + return self.mqs.pop() def _newmq(self, priority): + """ Factory for creating memory queues. """ return self.mqclass() def _newdq(self, priority): - return self.dqclass(join(self.dqdir, 'p%s' % (priority, ))) + """ Factory for creating disk queues. """ + path = join(self.dqdir, 'p%s' % (priority, )) + return self.dqclass(path) + + def _mq(self): + """ Create a new priority queue instance, with in-memory storage """ + return create_instance(self.pqclass, None, self.crawler, self._newmq, + serialize=False) def _dq(self): - activef = join(self.dqdir, 'active.json') - if exists(activef): - with open(activef) as f: - prios = json.load(f) - else: - prios = () - + """ Create a new priority queue instance, with disk storage """ + state = self._read_dqs_state(self.dqdir) q = create_instance(self.pqclass, None, self.crawler, self._newdq, - startprios=prios) + state, + serialize=True) if q: logger.info("Resuming crawl (%(queuesize)d requests scheduled)", {'queuesize': len(q)}, extra={'spider': self.spider}) return q def _dqdir(self, jobdir): + """ Return a folder name to keep disk queue state at """ if jobdir: dqdir = join(jobdir, 'requests.queue') if not exists(dqdir): os.makedirs(dqdir) return dqdir + + def _read_dqs_state(self, dqdir): + path = join(dqdir, 'active.json') + if not exists(path): + return () + with open(path) as f: + return json.load(f) + + def _write_dqs_state(self, dqdir, state): + with open(join(dqdir, 'active.json'), 'w') as f: + json.dump(state, f) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 6a9feb599..622f6bbc5 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -1,10 +1,10 @@ import hashlib import logging from collections import namedtuple -from six.moves.urllib.parse import urlparse from queuelib import PriorityQueue +from scrapy.utils.reqser import request_to_dict, request_from_dict from scrapy.core.downloader import Downloader from scrapy.http import Request from scrapy.signals import request_reached_downloader, request_left_downloader @@ -17,16 +17,6 @@ logger = logging.getLogger(__name__) SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT -def _get_request_meta(request): - if isinstance(request, dict): - return request.setdefault('meta', {}) - - if isinstance(request, Request): - return request.meta - - raise ValueError('Bad type of request "%s"' % (request.__class__, )) - - def _scheduler_slot_read(request, default=None): return request.meta.get(SCHEDULER_SLOT_META_KEY, default) @@ -37,24 +27,17 @@ def _scheduler_slot_write(request, slot): def _set_scheduler_slot(request): """ - >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://bar.com'}) - False - >>> _set_scheduler_slot({'url':'http://foo.com'}) == _set_scheduler_slot({'url':'http://foo.com'}) - True + >>> request = Request('http://example.com') + >>> _set_scheduler_slot(request) + 'example.com' + >>> _scheduler_slot_read(request) + 'example.com' """ - meta = _get_request_meta(request) - slot = meta.get(SCHEDULER_SLOT_META_KEY, None) - + slot = _scheduler_slot_read(request, None) if slot is not None: return slot - - if isinstance(request, dict): - url = request.get('url', None) - slot = urlparse(url).hostname or '' - elif isinstance(request, Request): - slot = urlparse_cached(request).hostname or '' - - meta[SCHEDULER_SLOT_META_KEY] = slot + slot = urlparse_cached(request).hostname or '' + _scheduler_slot_write(request, slot) return slot @@ -68,30 +51,25 @@ def _path_safe(text): return '-'.join([pathable_slot, unique_slot]) -class PrioritySlot(namedtuple("PrioritySlot", ["priority", "slot"])): - """ ``(priority, slot)`` tuple which uses a path-safe slot name - when converting to str """ +class _Priority(namedtuple("_Priority", ["priority", "slot"])): + """ Slot-specific priority. It is a hack - ``(priority, slot)`` tuple + which can be used instead of int priorities in queues: + + * they are ordered in the same way - order is still by priority value, + min(prios) works; + * str(p) representation is guaranteed to be different when slots + are different - this is important because str(p) is used to create + queue files on disk; + * they have readable str(p) representation which is safe + to use as a file name. + """ __slots__ = () def __str__(self): return '%s_%s' % (self.priority, _path_safe(str(self.slot))) -class PriorityAsTupleQueue(PriorityQueue): - """ - Python structures is not directly (de)serialized (to)from json. - We need this modified queue to transform custom structure (from)to - json serializable structures - """ - def __init__(self, qfactory, startprios=()): - startprios = [PrioritySlot(priority=p[0], slot=p[1]) - for p in startprios] - super(PriorityAsTupleQueue, self).__init__( - qfactory=qfactory, - startprios=startprios) - - -class SlotPriorityQueues(object): +class _SlotPriorityQueues(object): """ Container for multiple priority queues. """ def __init__(self, pqfactory, slot_startprios=None): """ @@ -134,44 +112,78 @@ class SlotPriorityQueues(object): return slot in self.pqueues -class DownloaderAwarePriorityQueue(object): - - _DOWNLOADER_AWARE_PQ_ID = 'DOWNLOADER_AWARE_PQ_ID' +class ScrapyPriorityQueue(PriorityQueue): + """ + PriorityQueue which works with scrapy.Request instances and + can optionally convert them to/from dicts before/after putting to a queue. + """ + def __init__(self, crawler, qfactory, startprios=(), serialize=False): + super(ScrapyPriorityQueue, self).__init__(qfactory, startprios) + self.serialize = serialize + self.spider = crawler.spider @classmethod - def from_crawler(cls, crawler, qfactory, startprios=None): - return cls(crawler, qfactory, startprios) + def from_crawler(cls, crawler, qfactory, startprios=(), serialize=False): + return cls(crawler, qfactory, startprios, serialize) - def __init__(self, crawler, qfactory, startprios=None): - ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP' - ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0) + def push(self, request, priority=0): + if self.serialize: + request = request_to_dict(request, self.spider) + super(ScrapyPriorityQueue, self).push(request, priority) - if ip_concurrency > 0: - raise ValueError('"%s" does not support setting %s' % (self.__class__, - ip_concurrency_key)) + def pop(self): + request = super(ScrapyPriorityQueue, self).pop() + if request and self.serialize: + request = request_from_dict(request, self.spider) + return request + + +class DownloaderAwarePriorityQueue(object): + """ PriorityQueue which takes Downlaoder activity in account: + domains (slots) with the least amount of active downloads are dequeued + first. + """ + _DOWNLOADER_AWARE_PQ_ID = '_DOWNLOADER_AWARE_PQ_ID' + + @classmethod + def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False): + return cls(crawler, qfactory, slot_startprios, serialize) + + def __init__(self, crawler, qfactory, slot_startprios=None, serialize=False): + if crawler.settings.getint('CONCURRENT_REQUESTS_PER_IP') != 0: + raise ValueError('"%s" does not support CONCURRENT_REQUESTS_PER_IP' + % (self.__class__,)) + + if slot_startprios and not isinstance(slot_startprios, dict): + raise ValueError("DownloaderAwarePriorityQueue accepts " + "``slot_startprios`` as a dict; %r instance " + "is passed. Most likely, it means the state is" + "created by an incompatible priority queue. " + "Only a crawl started with the same priority " + "queue class can be resumed." % + slot_startprios.__class__) + + slot_startprios = { + slot: [_Priority(p, slot) for p in startprios] + for slot, startprios in (slot_startprios or {}).items()} def pqfactory(startprios=()): - return PriorityAsTupleQueue(qfactory, startprios) - - if startprios and not isinstance(startprios, dict): - raise ValueError("DownloaderAwarePriorityQueue accepts " - "``startprios`` as a dict; %r instance is passed." - " Only a crawl started with the same priority " - "queue class can be resumed." % startprios.__class__) - self._slot_pqueues = SlotPriorityQueues(pqfactory, - slot_startprios=startprios) + return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize) + self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios) self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} crawler.signals.connect(self.on_response_download, signal=request_left_downloader) crawler.signals.connect(self.on_request_reached_downloader, signal=request_reached_downloader) + self.serialize = serialize + # There are two PriorityQueues at the same time (memory and disk-based), + # and they both listen to Downloader signals. To filter out signals + # coming from the other queue, each queue keeps track of its own + # requests using mark / unmark / check_mark methods. def mark(self, request): - meta = _get_request_meta(request) - if not isinstance(meta, dict): - raise ValueError('No meta attribute in %s' % (request, )) - meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) + request.meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) def check_mark(self, request): return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) @@ -194,7 +206,7 @@ class DownloaderAwarePriorityQueue(object): def push(self, request, priority): slot = _set_scheduler_slot(request) - priority_slot = PrioritySlot(priority=priority, slot=slot) + priority_slot = _Priority(priority=priority, slot=slot) self._slot_pqueues.push_slot(slot, request, priority_slot) if slot not in self._active_downloads: self._active_downloads[slot] = 0 @@ -206,8 +218,8 @@ class DownloaderAwarePriorityQueue(object): slot = _scheduler_slot_read(request) if slot not in self._active_downloads or self._active_downloads[slot] <= 0: - raise ValueError('Get response for wrong slot "%s"' % (slot, )) - self._active_downloads[slot] = self._active_downloads[slot] - 1 + raise ValueError('Got response for a wrong slot "%s"' % (slot, )) + self._active_downloads[slot] -= 1 if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues: del self._active_downloads[slot] @@ -220,7 +232,9 @@ class DownloaderAwarePriorityQueue(object): def close(self): self._active_downloads.clear() - return self._slot_pqueues.close() + active = self._slot_pqueues.close() + return {slot: [p.priority for p in startprios] + for slot, startprios in active.items()} def __len__(self): return len(self._slot_pqueues) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index ca004aedd..365b405cb 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -244,7 +244,7 @@ ROBOTSTXT_OBEY = False SCHEDULER = 'scrapy.core.scheduler.Scheduler' SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue' SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue' -SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue' +SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.ScrapyPriorityQueue' SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader' SPIDER_LOADER_WARN_ONLY = False diff --git a/scrapy/squeues.py b/scrapy/squeues.py index d2074a457..30cc926e5 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -7,6 +7,7 @@ from six.moves import cPickle as pickle from queuelib import queue + def _serializable_queue(queue_class, serialize, deserialize): class SerializableQueue(queue_class): @@ -22,6 +23,7 @@ def _serializable_queue(queue_class, serialize, deserialize): return SerializableQueue + def _pickle_serialize(obj): try: return pickle.dumps(obj, protocol=2) @@ -31,13 +33,14 @@ def _pickle_serialize(obj): except (pickle.PicklingError, AttributeError, TypeError) as e: raise ValueError(str(e)) -PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ + +PickleFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, _pickle_serialize, pickle.loads) -PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \ +PickleLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, _pickle_serialize, pickle.loads) -MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, \ +MarshalFifoDiskQueue = _serializable_queue(queue.FifoDiskQueue, marshal.dumps, marshal.loads) -MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, \ +MarshalLifoDiskQueue = _serializable_queue(queue.LifoDiskQueue, marshal.dumps, marshal.loads) FifoMemoryQueue = queue.FifoMemoryQueue LifoMemoryQueue = queue.LifoMemoryQueue From 443fb98a4776f4196662bb48918f1471758b7ae7 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Tue, 5 Mar 2019 12:44:07 +0000 Subject: [PATCH 072/140] Use downloader directly rename variable remove old write function remove unused imports remove old read function remove unused function use mock methods mock downloader close downloader add parse method use new PQ class create mock downloader use downloader directly remove mark/unmark mechanism --- scrapy/pqueues.py | 103 ++++++++++------------------------------ tests/test_scheduler.py | 87 +++++++++++++++++++++------------ 2 files changed, 81 insertions(+), 109 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 622f6bbc5..0681e6729 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -5,42 +5,11 @@ from collections import namedtuple from queuelib import PriorityQueue from scrapy.utils.reqser import request_to_dict, request_from_dict -from scrapy.core.downloader import Downloader -from scrapy.http import Request -from scrapy.signals import request_reached_downloader, request_left_downloader -from scrapy.utils.httpobj import urlparse_cached logger = logging.getLogger(__name__) -SCHEDULER_SLOT_META_KEY = Downloader.DOWNLOAD_SLOT - - -def _scheduler_slot_read(request, default=None): - return request.meta.get(SCHEDULER_SLOT_META_KEY, default) - - -def _scheduler_slot_write(request, slot): - request.meta[SCHEDULER_SLOT_META_KEY] = slot - - -def _set_scheduler_slot(request): - """ - >>> request = Request('http://example.com') - >>> _set_scheduler_slot(request) - 'example.com' - >>> _scheduler_slot_read(request) - 'example.com' - """ - slot = _scheduler_slot_read(request, None) - if slot is not None: - return slot - slot = urlparse_cached(request).hostname or '' - _scheduler_slot_write(request, slot) - return slot - - def _path_safe(text): """ Return a filesystem-safe version of a string ``text`` """ pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' @@ -138,6 +107,25 @@ class ScrapyPriorityQueue(PriorityQueue): return request +class DownloaderInterface(object): + + def __init__(self, crawler): + self.downloader = crawler.engine.downloader + + def stats(self, possible_slots): + return [(self._active_downloads(slot), slot) + for slot in possible_slots] + + def get_slot_key(self, request): + return self.downloader._get_slot_key(request, None) + + def _active_downloads(self, slot): + """ Return a number of requests in a Downloader for a given slot """ + if slot not in self.downloader.slots: + return 0 + return len(self.downloader.slots[slot].active) + + class DownloaderAwarePriorityQueue(object): """ PriorityQueue which takes Downlaoder activity in account: domains (slots) with the least amount of active downloads are dequeued @@ -170,68 +158,25 @@ class DownloaderAwarePriorityQueue(object): def pqfactory(startprios=()): return ScrapyPriorityQueue(crawler, qfactory, startprios, serialize) self._slot_pqueues = _SlotPriorityQueues(pqfactory, slot_startprios) - - self._active_downloads = {slot: 0 for slot in self._slot_pqueues.pqueues} - crawler.signals.connect(self.on_response_download, - signal=request_left_downloader) - crawler.signals.connect(self.on_request_reached_downloader, - signal=request_reached_downloader) self.serialize = serialize - - # There are two PriorityQueues at the same time (memory and disk-based), - # and they both listen to Downloader signals. To filter out signals - # coming from the other queue, each queue keeps track of its own - # requests using mark / unmark / check_mark methods. - def mark(self, request): - request.meta[self._DOWNLOADER_AWARE_PQ_ID] = id(self) - - def check_mark(self, request): - return request.meta.get(self._DOWNLOADER_AWARE_PQ_ID, None) == id(self) - - def unmark(self, request): - del request.meta[self._DOWNLOADER_AWARE_PQ_ID] + self._downloader_interface = DownloaderInterface(crawler) def pop(self): - slots = [(active_downloads, slot) - for slot, active_downloads in self._active_downloads.items() - if slot in self._slot_pqueues] + stats = self._downloader_interface.stats(self._slot_pqueues.pqueues) - if not slots: + if not stats: return - slot = min(slots)[1] + slot = min(stats)[1] request = self._slot_pqueues.pop_slot(slot) - self.mark(request) return request def push(self, request, priority): - slot = _set_scheduler_slot(request) + slot = self._downloader_interface.get_slot_key(request) priority_slot = _Priority(priority=priority, slot=slot) self._slot_pqueues.push_slot(slot, request, priority_slot) - if slot not in self._active_downloads: - self._active_downloads[slot] = 0 - - def on_response_download(self, request, spider): - if not self.check_mark(request): - return - self.unmark(request) - - slot = _scheduler_slot_read(request) - if slot not in self._active_downloads or self._active_downloads[slot] <= 0: - raise ValueError('Got response for a wrong slot "%s"' % (slot, )) - self._active_downloads[slot] -= 1 - if self._active_downloads[slot] == 0 and slot not in self._slot_pqueues: - del self._active_downloads[slot] - - def on_request_reached_downloader(self, request, spider): - if not self.check_mark(request): - return - - slot = _scheduler_slot_read(request) - self._active_downloads[slot] = self._active_downloads.get(slot, 0) + 1 def close(self): - self._active_downloads.clear() active = self._slot_pqueues.close() return {slot: [p.priority for p in startprios] for slot, startprios in active.items()} diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 1bcc1e5a8..75c0b7530 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -1,20 +1,50 @@ import shutil import tempfile import unittest +import collections from twisted.internet import defer from twisted.trial.unittest import TestCase from scrapy.crawler import Crawler +from scrapy.core.downloader import Downloader from scrapy.core.scheduler import Scheduler from scrapy.http import Request -from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write -from scrapy.signals import request_reached_downloader, request_left_downloader from scrapy.spiders import Spider +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler from tests.mockserver import MockServer +MockEngine = collections.namedtuple('MockEngine', ['downloader']) +MockSlot = collections.namedtuple('MockSlot', ['active']) + + +class MockDownloader: + def __init__(self): + self.slots = dict() + + def _set_slot_key(self, slot, request, spider): + request.meta[Downloader.DOWNLOAD_SLOT] = slot + + def _get_slot_key(self, request, spider): + if Downloader.DOWNLOAD_SLOT in request.meta: + return request.meta[Downloader.DOWNLOAD_SLOT] + + return urlparse_cached(request).hostname or '' + + def increment(self, slot_key): + slot = self.slots.setdefault(slot_key, MockSlot(active=list())) + slot.active.append(1) + + def decrement(self, slot_key): + slot = self.slots.get(slot_key) + slot.active.pop() + + def close(self): + pass + + class MockCrawler(Crawler): def __init__(self, priority_queue_cls, jobdir): @@ -27,6 +57,7 @@ class MockCrawler(Crawler): DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter' ) super(MockCrawler, self).__init__(Spider, settings) + self.engine = MockEngine(downloader=MockDownloader()) class SchedulerHandler: @@ -42,6 +73,7 @@ class SchedulerHandler: def close_scheduler(self): self.scheduler.close('finished') self.mock_crawler.stop() + self.mock_crawler.engine.downloader.close() def setUp(self): self.create_scheduler() @@ -147,11 +179,11 @@ class BaseSchedulerOnDiskTester(SchedulerHandler): class TestSchedulerInMemory(BaseSchedulerInMemoryTester, unittest.TestCase): - priority_queue_cls = 'queuelib.PriorityQueue' + priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): - priority_queue_cls = 'queuelib.PriorityQueue' + priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' _SLOTS = [("http://foo.com/a", 'a'), @@ -172,7 +204,7 @@ class TestMigration(unittest.TestCase): def _migration(self, tmp_dir): prev_scheduler_handler = SchedulerHandler() - prev_scheduler_handler.priority_queue_cls = 'queuelib.PriorityQueue' + prev_scheduler_handler.priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' prev_scheduler_handler.jobdir = tmp_dir prev_scheduler_handler.create_scheduler() @@ -196,30 +228,25 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): + downloader = self.mock_crawler.engine.downloader for url, slot in _SLOTS: request = Request(url) - _scheduler_slot_write(request, slot) + downloader._set_slot_key(slot, request, None) self.scheduler.enqueue_request(request) slots = list() requests = list() while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - slots.append(_scheduler_slot_read(request)) - self.mock_crawler.signals.send_catch_log( - signal=request_reached_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + slots.append(slot) + downloader.increment(slot) requests.append(request) self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log( - signal=request_left_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + self.mock_crawler.engine.downloader.decrement(slot) unique_slots = len(set(s for _, s in _SLOTS)) for i in range(0, len(_SLOTS), unique_slots): @@ -239,9 +266,11 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): + downloader = self.mock_crawler.engine.downloader + for url, slot in _SLOTS: request = Request(url) - _scheduler_slot_write(request, slot) + downloader._set_slot_key(slot, request, None) self.scheduler.enqueue_request(request) self.close_scheduler() @@ -249,27 +278,22 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, slots = [] requests = [] + downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() - slots.append(_scheduler_slot_read(request)) - self.mock_crawler.signals.send_catch_log( - signal=request_reached_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + slots.append(slot) + downloader.increment(slot) requests.append(request) - self.assertEqual(self.scheduler.mqs._active_downloads, {}) self.assertEqual(len(slots), len(_SLOTS)) for request in requests: - self.mock_crawler.signals.send_catch_log( - signal=request_left_downloader, - request=request, - spider=self.spider - ) + slot = downloader._get_slot_key(request, None) + downloader.decrement(slot) _is_slots_unique(_SLOTS, slots) + self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) class StartUrlsSpider(Spider): @@ -277,6 +301,9 @@ class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls + def parse(self, response): + pass + class TestIntegrationWithDownloaderAwareOnDisk(TestCase): def setUp(self): From 989bba6cb340fcc1ddb32e75ade567864d8b3884 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Thu, 7 Mar 2019 09:00:14 +0000 Subject: [PATCH 073/140] Revert "new signal" This reverts commit 646164fd7d6dd52061804d2df7424cff929bf739. remove tests Revert "emit new signal" This reverts commit fcde0c6880678957a76af6083b6248f430a00fcf. Revert "documentation for new signal" This reverts commit 8aeb9f696ece95c16499a96767a7afa3d9c4abf4. --- docs/topics/signals.rst | 17 --------- scrapy/core/downloader/__init__.py | 3 -- scrapy/signals.py | 1 - tests/test_request_left.py | 59 ------------------------------ 4 files changed, 80 deletions(-) delete mode 100644 tests/test_request_left.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f13e8270c..ff07b9d55 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -295,23 +295,6 @@ request_reached_downloader :param spider: the spider that yielded the request :type spider: :class:`~scrapy.spiders.Spider` object -request_left_downloader ---------------------------- - -.. signal:: request_left_downloader -.. function:: request_left_downloader(request, spider) - - Sent when a :class:`~scrapy.http.Request` left downloader even in case of - failure. - - The signal does not support returning deferreds from their handlers. - - :param request: the request that reached downloader - :type request: :class:`~scrapy.http.Request` object - - :param spider: the spider that yielded the request - :type spider: :class:`~scrapy.spiders.Spider` object - response_received ----------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index d856a2f37..4695d75f4 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -188,9 +188,6 @@ class Downloader(object): def finish_transferring(_): slot.transferring.remove(request) self._process_queue(spider, slot) - self.signals.send_catch_log(signal=signals.request_left_downloader, - request=request, - spider=spider) return _ return dfd.addBoth(finish_transferring) diff --git a/scrapy/signals.py b/scrapy/signals.py index 2ea986b8c..c0e4bb74e 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -14,7 +14,6 @@ spider_error = object() request_scheduled = object() request_dropped = object() request_reached_downloader = object() -request_left_downloader = object() response_received = object() response_downloaded = object() item_scraped = object() diff --git a/tests/test_request_left.py b/tests/test_request_left.py deleted file mode 100644 index ddeca0499..000000000 --- a/tests/test_request_left.py +++ /dev/null @@ -1,59 +0,0 @@ -from twisted.internet import defer -from twisted.trial.unittest import TestCase -from scrapy.signals import request_left_downloader -from scrapy.spiders import Spider -from scrapy.utils.test import get_crawler -from tests.mockserver import MockServer - -class SignalCatcherSpider(Spider): - name = 'signal_catcher' - - def __init__(self, crawler, url, *args, **kwargs): - super(SignalCatcherSpider, self).__init__(*args, **kwargs) - crawler.signals.connect(self.on_response_download, - signal=request_left_downloader) - self.catched_times = 0 - self.start_urls = [url] - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = cls(crawler, *args, **kwargs) - return spider - - def on_response_download(self, request, spider): - self.catched_times = self.catched_times + 1 - - -class TestCatching(TestCase): - - def setUp(self): - self.mockserver = MockServer() - self.mockserver.__enter__() - - def tearDown(self): - self.mockserver.__exit__(None, None, None) - - @defer.inlineCallbacks - def test_success(self): - crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/status?n=200")) - self.assertEqual(crawler.spider.catched_times, 1) - - @defer.inlineCallbacks - def test_timeout(self): - crawler = get_crawler(SignalCatcherSpider, - {'DOWNLOAD_TIMEOUT': 0.1}) - yield crawler.crawl(self.mockserver.url("/delay?n=0.2")) - self.assertEqual(crawler.spider.catched_times, 1) - - @defer.inlineCallbacks - def test_disconnect(self): - crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl(self.mockserver.url("/drop")) - self.assertEqual(crawler.spider.catched_times, 1) - - @defer.inlineCallbacks - def test_noconnect(self): - crawler = get_crawler(SignalCatcherSpider) - yield crawler.crawl('http://thereisdefinetelynosuchdomain.com') - self.assertEqual(crawler.spider.catched_times, 1) From 8afffb7234b282dd8bd28eec2e4eb8e3f86b5723 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 22 Mar 2019 09:12:23 +0000 Subject: [PATCH 074/140] Tests Cleanup add doctest for function no need in this variables move common assertion inside function rename variable rename variables rename function use function this is not a method of public API correct name for test Update docs/topics/settings.rst Co-Authored-By: whalebot-helmsman --- docs/topics/settings.rst | 4 +- tests/test_scheduler.py | 82 ++++++++++++++++++++++------------------ 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 6e13e64d6..cf454f4ec 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1146,9 +1146,9 @@ Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` Type of priority queue used by scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. -``scrapy.pqueues.DownloaderAwarePriorityQueue`` is works better than +``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different -domains in parallel. But ``scrapy.pqueues.DownloaderAwarePriorityQueue`` +domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue`` does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`. .. setting:: SPIDER_CONTRACTS diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 75c0b7530..eaf748d35 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -24,9 +24,6 @@ class MockDownloader: def __init__(self): self.slots = dict() - def _set_slot_key(self, slot, request, spider): - request.meta[Downloader.DOWNLOAD_SLOT] = slot - def _get_slot_key(self, request, spider): if Downloader.DOWNLOAD_SLOT in request.meta: return request.meta[Downloader.DOWNLOAD_SLOT] @@ -186,12 +183,12 @@ class TestSchedulerOnDisk(BaseSchedulerOnDiskTester, unittest.TestCase): priority_queue_cls = 'scrapy.pqueues.ScrapyPriorityQueue' -_SLOTS = [("http://foo.com/a", 'a'), - ("http://foo.com/b", 'a'), - ("http://foo.com/c", 'b'), - ("http://foo.com/d", 'b'), - ("http://foo.com/e", 'c'), - ("http://foo.com/f", 'c')] +_URLS_WITH_SLOTS = [("http://foo.com/a", 'a'), + ("http://foo.com/b", 'a'), + ("http://foo.com/c", 'b'), + ("http://foo.com/d", 'b'), + ("http://foo.com/e", 'c'), + ("http://foo.com/f", 'c')] class TestMigration(unittest.TestCase): @@ -228,37 +225,52 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): - downloader = self.mock_crawler.engine.downloader - for url, slot in _SLOTS: + for url, slot in _URLS_WITH_SLOTS: request = Request(url) - downloader._set_slot_key(slot, request, None) + request.meta[Downloader.DOWNLOAD_SLOT] = slot self.scheduler.enqueue_request(request) - slots = list() + downloader = self.mock_crawler.engine.downloader + dequeued_slots = list() requests = list() while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() slot = downloader._get_slot_key(request, None) - slots.append(slot) + dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) - self.assertEqual(len(slots), len(_SLOTS)) for request in requests: slot = downloader._get_slot_key(request, None) self.mock_crawler.engine.downloader.decrement(slot) - unique_slots = len(set(s for _, s in _SLOTS)) - for i in range(0, len(_SLOTS), unique_slots): - part = slots[i:i + unique_slots] - self.assertEqual(len(part), len(set(part))) + self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), + dequeued_slots)) -def _is_slots_unique(base_slots, result_slots): - unique_slots = len(set(s for _, s in base_slots)) - for i in range(0, len(result_slots), unique_slots): - part = result_slots[i:i + unique_slots] - assert len(part) == len(set(part)) +def _is_scheduling_fair(enqueued_slots, dequeued_slots): + """ + We enqueued same number of requests for every slot. + Assert correct order, e.g. + + >>> enqueued = ['a', 'b', 'c'] * 2 + >>> correct = ['a', 'c', 'b', 'b', 'a', 'c'] + >>> incorrect = ['a', 'a', 'b', 'c', 'c', 'b'] + >>> _is_scheduling_fair(enqueued, correct) + True + >>> _is_scheduling_fair(enqueued, incorrect) + False + """ + if len(dequeued_slots) != len(enqueued_slots): + return False + + slots_number = len(set(enqueued_slots)) + for i in range(0, len(dequeued_slots), slots_number): + part = dequeued_slots[i:i + slots_number] + if len(part) != len(set(part)): + return False + + return True class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, @@ -266,33 +278,31 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' def test_logic(self): - downloader = self.mock_crawler.engine.downloader - for url, slot in _SLOTS: + for url, slot in _URLS_WITH_SLOTS: request = Request(url) - downloader._set_slot_key(slot, request, None) + request.meta[Downloader.DOWNLOAD_SLOT] = slot self.scheduler.enqueue_request(request) self.close_scheduler() self.create_scheduler() - slots = [] + dequeued_slots = list() requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() slot = downloader._get_slot_key(request, None) - slots.append(slot) + dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) - self.assertEqual(len(slots), len(_SLOTS)) - for request in requests: slot = downloader._get_slot_key(request, None) downloader.decrement(slot) - _is_slots_unique(_SLOTS, slots) + self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), + dequeued_slots)) self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) @@ -305,7 +315,7 @@ class StartUrlsSpider(Spider): pass -class TestIntegrationWithDownloaderAwareOnDisk(TestCase): +class TestIntegrationWithDownloaderAwareInMemory(TestCase): def setUp(self): self.crawler = get_crawler( StartUrlsSpider, @@ -322,10 +332,10 @@ class TestIntegrationWithDownloaderAwareOnDisk(TestCase): with MockServer() as mockserver: url = mockserver.url("/status?n=200", is_secure=False) - slots = [url] * 6 - yield self.crawler.crawl(slots) + start_urls = [url] * 6 + yield self.crawler.crawl(start_urls) self.assertEqual(self.crawler.stats.get_value('downloader/response_count'), - len(slots)) + len(start_urls)) class TestIncompatibility(unittest.TestCase): From df574de8cc5c58618f6075ca3afb14059a9e30ed Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Sat, 23 Mar 2019 00:54:39 +0800 Subject: [PATCH 075/140] improve tests and fix some lint warnings (#6) * refactor downloader-aware test cases * fix lint * add doctest for _path_safe * remove unused code * better doctest --- scrapy/pqueues.py | 12 +++++++-- tests/test_scheduler.py | 57 ++++++++++++++++------------------------- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index 0681e6729..6ecd1b51a 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -11,7 +11,16 @@ logger = logging.getLogger(__name__) def _path_safe(text): - """ Return a filesystem-safe version of a string ``text`` """ + """ + Return a filesystem-safe version of a string ``text`` + + >>> _path_safe('simple.org').startswith('simple.org') + True + >>> _path_safe('dash-underscore_.org').startswith('dash-underscore_.org') + True + >>> _path_safe('some@symbol?').startswith('some_symbol_') + True + """ pathable_slot = "".join([c if c.isalnum() or c in '-._' else '_' for c in text]) # as we replace some letters we can get collision for different slots @@ -131,7 +140,6 @@ class DownloaderAwarePriorityQueue(object): domains (slots) with the least amount of active downloads are dequeued first. """ - _DOWNLOADER_AWARE_PQ_ID = '_DOWNLOADER_AWARE_PQ_ID' @classmethod def from_crawler(cls, crawler, qfactory, slot_startprios=None, serialize=False): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index eaf748d35..e0e3600e5 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -20,7 +20,7 @@ MockEngine = collections.namedtuple('MockEngine', ['downloader']) MockSlot = collections.namedtuple('MockSlot', ['active']) -class MockDownloader: +class MockDownloader(object): def __init__(self): self.slots = dict() @@ -57,7 +57,7 @@ class MockCrawler(Crawler): self.engine = MockEngine(downloader=MockDownloader()) -class SchedulerHandler: +class SchedulerHandler(object): priority_queue_cls = None jobdir = None @@ -220,34 +220,6 @@ class TestMigration(unittest.TestCase): self._migration(self.tmpdir) -class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester, - unittest.TestCase): - priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' - - def test_logic(self): - for url, slot in _URLS_WITH_SLOTS: - request = Request(url) - request.meta[Downloader.DOWNLOAD_SLOT] = slot - self.scheduler.enqueue_request(request) - - downloader = self.mock_crawler.engine.downloader - dequeued_slots = list() - requests = list() - while self.scheduler.has_pending_requests(): - request = self.scheduler.next_request() - slot = downloader._get_slot_key(request, None) - dequeued_slots.append(slot) - downloader.increment(slot) - requests.append(request) - - for request in requests: - slot = downloader._get_slot_key(request, None) - self.mock_crawler.engine.downloader.decrement(slot) - - self.assertTrue(_is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), - dequeued_slots)) - - def _is_scheduling_fair(enqueued_slots, dequeued_slots): """ We enqueued same number of requests for every slot. @@ -273,31 +245,33 @@ def _is_scheduling_fair(enqueued_slots, dequeued_slots): return True -class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, - unittest.TestCase): +class DownloaderAwareSchedulerTestMixin(object): priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue' + reopen = False def test_logic(self): - for url, slot in _URLS_WITH_SLOTS: request = Request(url) request.meta[Downloader.DOWNLOAD_SLOT] = slot self.scheduler.enqueue_request(request) - self.close_scheduler() - self.create_scheduler() + if self.reopen: + self.close_scheduler() + self.create_scheduler() dequeued_slots = list() requests = [] downloader = self.mock_crawler.engine.downloader while self.scheduler.has_pending_requests(): request = self.scheduler.next_request() + # pylint: disable=protected-access slot = downloader._get_slot_key(request, None) dequeued_slots.append(slot) downloader.increment(slot) requests.append(request) for request in requests: + # pylint: disable=protected-access slot = downloader._get_slot_key(request, None) downloader.decrement(slot) @@ -306,10 +280,23 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester, self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0) +class TestSchedulerWithDownloaderAwareInMemory(DownloaderAwareSchedulerTestMixin, + BaseSchedulerInMemoryTester, + unittest.TestCase): + pass + + +class TestSchedulerWithDownloaderAwareOnDisk(DownloaderAwareSchedulerTestMixin, + BaseSchedulerOnDiskTester, + unittest.TestCase): + reopen = True + + class StartUrlsSpider(Spider): def __init__(self, start_urls): self.start_urls = start_urls + super(StartUrlsSpider, self).__init__(start_urls) def parse(self, response): pass From 31b8a6b33aed9e77a4d37a5c83b1545202207cad Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 25 Mar 2019 08:53:15 +0000 Subject: [PATCH 076/140] report warnings --- tests/test_crawler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 268948a70..d9ec9ee8d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,5 +1,4 @@ import logging -import tempfile import warnings from twisted.internet import defer @@ -37,7 +36,11 @@ class CrawlerTestCase(BaseCrawlerTest): self.assertIsInstance(spiders, sl_cls) self.crawler.spiders - self.assertEqual(len(w), 1, "Warn deprecated access only once") + is_one_warning = len(w) == 1 + if not is_one_warning: + for warning in w: + print(warning) + self.assertTrue(is_one_warning, "Warn deprecated access only once") def test_populate_spidercls_settings(self): spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'} From 73e4ff5304d273404a147d06726a8ae8cae1c925 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Mon, 25 Mar 2019 13:48:58 +0000 Subject: [PATCH 077/140] report warnings --- tests/test_crawler.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 8c4bbe0d9..e811c5757 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -182,8 +182,12 @@ class CrawlerRunnerTestCase(BaseCrawlerTest): 'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader' }) self.assertIsInstance(runner.spider_loader, CustomSpiderLoader) - self.assertEqual(len(w), 1) + is_one_warning = len(w) == 1 + if not is_one_warning: + for warning in w: + print(warning) self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message)) + self.assertTrue(is_one_warning) def test_crawl_rejects_spider_objects(self): with raises(ValueError): From 845bae6637239c859c9952c23f42902e36d10f6b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 27 Mar 2019 08:49:19 +0000 Subject: [PATCH 078/140] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 37f7a8748..64c8883b1 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -42,7 +42,7 @@ efficient broad crawl. Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== -Default scrapy's scheduler priority queue is ``'queuelib.PriorityQueue'``. +Default scrapy's scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. It works best during single domain crawl. And it does not work well with crawling many different domains in parallel From 46b9ab0c58354deb1045c20f3bc061526d69f356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:28:36 +0000 Subject: [PATCH 079/140] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 64c8883b1..68a24a4d2 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -42,7 +42,7 @@ efficient broad crawl. Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== -Default scrapy's scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. +Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. It works best during single domain crawl. And it does not work well with crawling many different domains in parallel From e3df6be360a58f016e31d5bfa2e04cd2e5d1965b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:28:52 +0000 Subject: [PATCH 080/140] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 68a24a4d2..b149d7f4a 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -43,7 +43,7 @@ Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. -It works best during single domain crawl. And it does not work well with crawling +It works best during single-domain crawl. It does not work well with crawling many different domains in parallel To apply recommended priority queue use:: From bd228f1d962c7f4759536d8cda278857de7d5234 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:29:04 +0000 Subject: [PATCH 081/140] Update docs/topics/broad-crawls.rst Co-Authored-By: whalebot-helmsman --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index b149d7f4a..a01f28248 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -46,7 +46,7 @@ Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQ It works best during single-domain crawl. It does not work well with crawling many different domains in parallel -To apply recommended priority queue use:: +To apply the recommended priority queue use:: SCHEDULER_PRIORITY_QUEUE = 'scrapy.pqueues.DownloaderAwarePriorityQueue' From 1ee99e1f4240af6a7a72fe7c58b89d7bce1cd09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 10:29:15 +0000 Subject: [PATCH 082/140] Update docs/topics/settings.rst Co-Authored-By: whalebot-helmsman --- docs/topics/settings.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index ed94146f4..4a5439bfc 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1157,7 +1157,7 @@ SCHEDULER_PRIORITY_QUEUE ------------------------ Default: ``'scrapy.pqueues.ScrapyPriorityQueue'`` -Type of priority queue used by scheduler. Another available type is +Type of priority queue used by the scheduler. Another available type is ``scrapy.pqueues.DownloaderAwarePriorityQueue``. ``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different From 2b4bcfaf494073520e84bbf301d5141a2e19a3e6 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:30:26 +0000 Subject: [PATCH 083/140] remove comment --- scrapy/core/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index c385fafe1..9d0258db2 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -57,7 +57,6 @@ class Scheduler(object): dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) if pqclass is PriorityQueue: - # backwards compatibility warnings.warn("SCHEDULER_PRIORITY_QUEUE='queuelib.PriorityQueue'" " is no longer supported because of API changes; " "please use 'scrapy.pqueues.ScrapyPriorityQueue'", From 554d8728227a9ea96e5ea3a8a4fd782d42fdbd66 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:31:15 +0000 Subject: [PATCH 084/140] remove spacing --- scrapy/core/scheduler.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 9d0258db2..d87d2ffdc 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -77,13 +77,8 @@ class Scheduler(object): def open(self, spider): self.spider = spider - - # in-memory PriorityQueue instance self.mqs = self._mq() - - # on-disk PriorityQueue instance self.dqs = self._dq() if self.dqdir else None - return self.df.open() def close(self, reason): From f08f841d0bebd097358889c0c98f83f051828f15 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:35:49 +0000 Subject: [PATCH 085/140] remove small single use method --- scrapy/core/scheduler.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index d87d2ffdc..975aede0c 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -101,7 +101,7 @@ class Scheduler(object): return True def next_request(self): - request = self._mqpop() + request = self.mqs.pop() if request: self.stats.inc_value('scheduler/dequeued/memory', spider=self.spider) else: @@ -141,9 +141,6 @@ class Scheduler(object): if self.dqs: return self.dqs.pop() - def _mqpop(self): - return self.mqs.pop() - def _newmq(self, priority): """ Factory for creating memory queues. """ return self.mqclass() From ef743983a98ae0891abf9aca4c9b19cb44861c49 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:38:13 +0000 Subject: [PATCH 086/140] change wording --- docs/topics/broad-crawls.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index a01f28248..6e50c0bc7 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -39,7 +39,7 @@ you need to keep in mind when using Scrapy for doing broad crawls, along with concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. -Use proper :setting:`SCHEDULER_PRIORITY_QUEUE` +Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` ============================================== Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. @@ -96,7 +96,7 @@ When doing broad crawls you are often only interested in the crawl rates you get and any errors found. These stats are reported by Scrapy when using the ``INFO`` log level. In order to save CPU (and log storage requirements) you should not use ``DEBUG`` log level when preforming large broad crawls in -production. Using ``DEBUG`` level when developing your (broad) crawler may be +production. Using ``DEBUG`` level when developing your (broad) crawler may be fine though. To set the log level use:: From 1c6733454e14a3c237ed602b65ae5e0a8a78dee5 Mon Sep 17 00:00:00 2001 From: Vostretsov Nikita Date: Fri, 29 Mar 2019 10:44:55 +0000 Subject: [PATCH 087/140] added underlines --- docs/topics/broad-crawls.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 6e50c0bc7..b887b98af 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -40,7 +40,7 @@ concrete suggestions of Scrapy settings to tune in order to achieve an efficient broad crawl. Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` -============================================== +================================================= Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. It works best during single-domain crawl. It does not work well with crawling From 7acf4eec792f155a8b0e92c3bb1efa1ff5882ac8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 2 Apr 2019 18:36:03 +0200 Subject: [PATCH 088/140] Deprecate the scrapy.utils.gz.is_gzipped function --- scrapy/utils/gz.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index ec3949651..b3fb16b1e 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -9,6 +9,9 @@ from gzip import GzipFile import six import re +from scrapy.utils.decorators import deprecated + + # - Python>=3.5 GzipFile's read() has issues returning leftover # uncompressed data when input is corrupted # (regression or bug-fix compared to Python 3.4) @@ -53,6 +56,7 @@ def gunzip(data): _is_gzipped = re.compile(br'^application/(x-)?gzip\b', re.I).search _is_octetstream = re.compile(br'^(application|binary)/octet-stream\b', re.I).search +@deprecated def is_gzipped(response): """Return True if the response is gzipped, or False otherwise""" ctype = response.headers.get('Content-Type', b'') From 6336e1d1f31da8611a0b63dcb536529e9027d51b Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 00:54:46 -0400 Subject: [PATCH 089/140] grammar fix - delete unneeded apostrophe in "lets" --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 41e61542a..b2f952fe2 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -511,7 +511,7 @@ We can try extracting it in the shell:: 'Next ' This gets the anchor element, but we want the attribute ``href``. For that, -Scrapy supports a CSS extension that let's you select the attribute contents, +Scrapy supports a CSS extension that lets you select the attribute contents, like this:: >>> response.css('li.next a::attr(href)').get() From d711ecfc18a01084f74bb2b9dc01c8bcb4772580 Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 00:56:51 -0400 Subject: [PATCH 090/140] grammar fix - delete extra word "shell" --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index b2f952fe2..fc10adbe1 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -205,7 +205,7 @@ Extracting data --------------- The best way to learn how to extract data with Scrapy is trying selectors -using the shell :ref:`Scrapy shell `. Run:: +using the :ref:`Scrapy shell `. Run:: scrapy shell 'http://quotes.toscrape.com/page/1/' From 77e3695686d9a46841778248f4c1a2da336b54f7 Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 01:04:59 -0400 Subject: [PATCH 091/140] grammar fix - add apostrophe-s to browser --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index fc10adbe1..a97f96801 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -296,7 +296,7 @@ expressions`_:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. -You can use your browser developer tools to inspect the HTML and come up +You can use your browser's developer tools to inspect the HTML and come up with a selector (see section about :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for From a101d5fe5c215bfbb09732fb506d4cd016624d80 Mon Sep 17 00:00:00 2001 From: float13 <43447704+float13@users.noreply.github.com> Date: Fri, 5 Apr 2019 01:12:20 -0400 Subject: [PATCH 092/140] text edit - delete 2 extra words --- docs/intro/tutorial.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index a97f96801..8bd2d27dd 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -297,7 +297,7 @@ expressions`_:: In order to find the proper CSS selectors to use, you might find useful opening the response page from the shell in your web browser using ``view(response)``. You can use your browser's developer tools to inspect the HTML and come up -with a selector (see section about :ref:`topics-developer-tools`). +with a selector (see :ref:`topics-developer-tools`). `Selector Gadget`_ is also a nice tool to quickly find CSS selector for visually selected elements, which works in many browsers. From 3a493b60661760b26ebc9dd2f4c5c7e4b8df93c9 Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Fri, 5 Apr 2019 11:52:00 +0200 Subject: [PATCH 093/140] fix: do not catch system exceptions like KeyboardInterrupt --- scrapy/contracts/__init__.py | 2 +- scrapy/core/spidermw.py | 2 +- scrapy/utils/defer.py | 4 ++-- scrapy/utils/misc.py | 2 +- tests/mockserver.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 259220a72..536bbdafb 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -94,7 +94,7 @@ class ContractsManager(object): try: output = cb(response) output = list(iterate_spider_output(output)) - except: + except Exception: case = _create_testcase(method, 'callback') results.addError(case, sys.exc_info()) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index e07f76bdf..b5f9837ff 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -49,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager): .format(fname(method), type(result))) except _InvalidOutput: raise - except: + except Exception: return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index bcf209511..69d621830 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -48,7 +48,7 @@ def mustbe_deferred(f, *args, **kw): # exception in Scrapy - see #125 except IgnoreRequest as e: return defer_fail(failure.Failure(e)) - except: + except Exception: return defer_fail(failure.Failure()) else: return defer_result(result) @@ -102,5 +102,5 @@ def iter_errback(iterable, errback, *a, **kw): yield next(it) except StopIteration: break - except: + except Exception: errback(failure.Failure(), *a, **kw) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 6de36d45c..ddaa7f7bf 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -86,7 +86,7 @@ def extract_regex(regex, text, encoding='utf-8'): try: strings = [regex.search(text).group('extract')] # named group - except: + except Exception: strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) diff --git a/tests/mockserver.py b/tests/mockserver.py index bf62fe907..3fa4bc0f0 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -177,7 +177,7 @@ class Root(Resource): try: from tests import tests_datadir self.putChild(b"files", File(os.path.join(tests_datadir, 'test_site/files/'))) - except: + except Exception: pass self.putChild(b"redirect-to", RedirectTo()) From 35ce92a4199b29a30e58026880b45002a1e4591e Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Fri, 5 Apr 2019 11:43:21 -0300 Subject: [PATCH 094/140] fix typo (Response -> Request) check docs for more information https://github.com/scrapy/scrapy/blob/b5c552d17ff9e9629434712c3d0595c02853bcfc/docs/topics/spider-middleware.rst --- scrapy/templates/project/module/middlewares.py.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 5debe1cd2..97b5db2e1 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -39,7 +39,7 @@ class ${ProjectName}SpiderMiddleware(object): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. - # Should return either None or an iterable of Response, dict + # Should return either None or an iterable of Request, dict # or Item objects. pass From a8f83ab9675ec4f0bcb90b2eb3f06c593f32732f Mon Sep 17 00:00:00 2001 From: Kostis Anagnostopoulos Date: Sat, 6 Apr 2019 14:58:32 +0200 Subject: [PATCH 095/140] doc: document LOGSTATS_INTERVAL setting --- docs/topics/settings.rst | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 062c4b2ca..145dcc136 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -897,6 +897,16 @@ Default: ``False`` If ``True``, the logs will just contain the root path. If it is set to ``False`` then it displays the component responsible for the log output +.. setting:: LOGSTATS_INTERVAL + +LOGSTATS_INTERVAL +----------------- + +Default: ``60.0`` + +The interval (in seconds) between each logging printout of the stats +by :class:`~extensions.logstats.LogStats`. + .. setting:: MEMDEBUG_ENABLED MEMDEBUG_ENABLED From aa46e1995cd5cb1099aba17535372b538bd656b3 Mon Sep 17 00:00:00 2001 From: Maram Sumanth Date: Sun, 7 Apr 2019 00:33:40 +0530 Subject: [PATCH 096/140] [MRG+1] Show elapsed time in statscollector (#3638) * Update corestats.py * Update corestats.py * corrected tests * Update corestats.py * Update scrapy/extensions/corestats.py --- scrapy/extensions/corestats.py | 6 +++++- tests/test_closespider.py | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py index 3d9a307b7..8cc5e18ac 100644 --- a/scrapy/extensions/corestats.py +++ b/scrapy/extensions/corestats.py @@ -24,7 +24,11 @@ class CoreStats(object): self.stats.set_value('start_time', datetime.datetime.utcnow(), spider=spider) def spider_closed(self, spider, reason): - self.stats.set_value('finish_time', datetime.datetime.utcnow(), spider=spider) + finish_time = datetime.datetime.utcnow() + elapsed_time = finish_time - self.stats.get_value('start_time') + elapsed_time_seconds = elapsed_time.total_seconds() + self.stats.set_value('elapsed_time_seconds', elapsed_time_seconds, spider=spider) + self.stats.set_value('finish_time', finish_time, spider=spider) self.stats.set_value('finish_reason', reason, spider=spider) def item_scraped(self, item, spider): diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 0eb1b7944..4a56425b7 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -53,9 +53,5 @@ class TestCloseSpider(TestCase): yield crawler.crawl(total=1000000, mockserver=self.mockserver) reason = crawler.spider.meta['close_reason'] self.assertEqual(reason, 'closespider_timeout') - stats = crawler.stats - start = stats.get_value('start_time') - stop = stats.get_value('finish_time') - diff = stop - start - total_seconds = diff.seconds + diff.microseconds + total_seconds = crawler.stats.get_value('elapsed_time_seconds') self.assertTrue(total_seconds >= close_on) From e6048d55f905ca0be2b32f9b566d257f34752c71 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 9 Apr 2019 17:34:20 +0530 Subject: [PATCH 097/140] changes parameter name --- scrapy/extensions/httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 1b5e05b1b..35c77add8 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -31,7 +31,7 @@ class DummyPolicy(object): def should_cache_response(self, response, request): return response.status not in self.ignore_http_codes - def is_cached_response_fresh(self, response, request): + def is_cached_response_fresh(self, cachedresponse, request): return True def is_cached_response_valid(self, cachedresponse, response, request): From 4cfdc14974313f23d7bb8be9311195f9cfa74968 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Tue, 9 Apr 2019 17:52:02 +0530 Subject: [PATCH 098/140] fixes a link in comment --- scrapy/extensions/httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 1b5e05b1b..03bba1530 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -70,7 +70,7 @@ class RFC2616Policy(object): return True def should_cache_response(self, response, request): - # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1 # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 # Status code 206 is not included because cache can not deal with partial contents cc = self._parse_cachecontrol(response) From d27c2c68ba4f201c18255a6ec8a735f140b13773 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 12:56:50 +0200 Subject: [PATCH 099/140] Wrap scrapy check in environment --- scrapy/commands/check.py | 21 +++++++++++---------- scrapy/utils/misc.py | 27 +++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b8a9ef989..b29587fa7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -6,7 +6,7 @@ from unittest import TextTestRunner, TextTestResult as _TextTestResult from scrapy.commands import ScrapyCommand from scrapy.contracts import ContractsManager -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, set_environ from scrapy.utils.conf import build_component_list @@ -68,16 +68,17 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - for spidername in args or spider_loader.list(): - spidercls = spider_loader.load(spidername) - spidercls.start_requests = lambda s: conman.from_spider(s, result) + with set_environ(SCRAPY_CHECK=True): + for spidername in args or spider_loader.list(): + spidercls = spider_loader.load(spidername) + spidercls.start_requests = lambda s: conman.from_spider(s, result) - tested_methods = conman.tested_methods_from_spidercls(spidercls) - if opts.list: - for method in tested_methods: - contract_reqs[spidercls.name].append(method) - elif tested_methods: - self.crawler_process.crawl(spidercls) + tested_methods = conman.tested_methods_from_spidercls(spidercls) + if opts.list: + for method in tested_methods: + contract_reqs[spidercls.name].append(method) + elif tested_methods: + self.crawler_process.crawl(spidercls) # start checks if opts.list: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ddaa7f7bf..7a2cd18ea 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -1,6 +1,8 @@ """Helper functions which don't fit anywhere else""" +import os import re import hashlib +from contextlib import contextmanager from importlib import import_module from pkgutil import iter_modules @@ -10,7 +12,6 @@ from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem - _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes @@ -40,7 +41,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot+1:] + module, name = path[:dot], path[dot + 1:] mod = import_module(module) try: @@ -85,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): @@ -142,3 +143,21 @@ def create_instance(objcls, settings, crawler, *args, **kwargs): return objcls.from_settings(settings, *args, **kwargs) else: return objcls(*args, **kwargs) + + +@contextmanager +def set_environ(**kwargs): + """Temporarily set environment variables inside the context manager and + fully restore previous environment afterwards + """ + + original_env = {k: os.environ.get(k) for k in kwargs} + os.environ.update(kwargs) + try: + yield + finally: + for k, v in original_env: + if v is None: + del os.environ[k] + else: + os.environ[k] = v From 50730ed2280dec6384986d34999cd277d7568ff9 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 13:01:01 +0200 Subject: [PATCH 100/140] Try it with a string --- scrapy/commands/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index b29587fa7..ab73e85e7 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -68,7 +68,7 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader - with set_environ(SCRAPY_CHECK=True): + with set_environ(SCRAPY_CHECK='true'): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) spidercls.start_requests = lambda s: conman.from_spider(s, result) From 07adca34e1378b11dff9e3f11d3760c54f5fa1ef Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Wed, 10 Apr 2019 13:01:46 +0200 Subject: [PATCH 101/140] Fix 'Too many values to unpack' --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 7a2cd18ea..cdd5a11c9 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -156,7 +156,7 @@ def set_environ(**kwargs): try: yield finally: - for k, v in original_env: + for k, v in original_env.items(): if v is None: del os.environ[k] else: From fbb42fe14ed23aaba37b42d12c7adbf513f9089e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 17 Apr 2019 08:25:22 +0200 Subject: [PATCH 102/140] Cover PEP 257 in the documentation policies --- docs/contributing.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index aac0f4496..c31a17609 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -167,8 +167,9 @@ Documentation policies For reference documentation of API members (classes, methods, etc.) use docstrings and make sure that the Sphinx documentation uses the autodoc_ -extension to pull the docstrings. API reference documentation should be -IDE-friendly: short, to the point, and it may provide short examples. +extension to pull the docstrings. API reference documentation should follow +docstring conventions (`PEP 257`_) and be IDE-friendly: short, to the point, +and it may provide short examples. Other types of documentation, such as tutorials or topics, should be covered in files within the ``docs/`` directory. This includes documentation that is @@ -237,5 +238,6 @@ And their unit-tests are in:: .. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues +.. _PEP 257: https://www.python.org/dev/peps/pep-0257/ .. _pull request: https://help.github.com/send-pull-requests/ .. _tox: https://pypi.python.org/pypi/tox From 5a6fb3daa6e6a15effe9377dbcc85e67bec9aec7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 29 Mar 2019 17:10:16 +0100 Subject: [PATCH 103/140] Use pytest-xdist --- docs/contributing.rst | 26 ++++++++++++++++++++++++++ tests/requirements-py2.txt | 5 +++-- tests/requirements-py3.txt | 5 +++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index aac0f4496..b462ae331 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -205,6 +205,29 @@ To run a specific test (say ``tests/test_loader.py``) use: ``tox -- tests/test_loader.py`` +To run the tests on a specific tox_ environment, use ``-e `` with an +environment name from ``tox.ini``. For example, to run the tests with Python +3.6 use:: + + tox -e py36 + +You can also specify a comma-separated list of environmets, and use `tox’s +parallel mode`_ to run the tests on multiple environments in parallel:: + + tox -e py27,py36 -p auto + +To pass command-line options to pytest_, add them after ``--`` in your call to +tox_. Using ``--`` overrides the default positional arguments defined in +``tox.ini``, so you must include those default positional arguments +(``scrapy tests``) after ``--`` as well:: + + tox -- scrapy tests -x # stop after first failure + +You can also use the `pytest-xdist`_ plugin. For example, to run all tests on +the Python 3.6 tox_ environment using all your CPU cores:: + + tox -e py36 -- scrapy tests -n auto + To see coverage report install `coverage`_ (``pip install coverage``) and run: ``coverage report`` @@ -238,4 +261,7 @@ And their unit-tests are in:: .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _pull request: https://help.github.com/send-pull-requests/ +.. _pytest: https://docs.pytest.org/en/latest/usage.html +.. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html .. _tox: https://pypi.python.org/pypi/tox +.. _tox’s parallel mode: https://tox.readthedocs.io/en/latest/example/basic.html#parallel-mode diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt index 790f29d34..be809b151 100644 --- a/tests/requirements-py2.txt +++ b/tests/requirements-py2.txt @@ -2,9 +2,10 @@ mock mitmproxy==0.10.1 netlib==0.10.1 -pytest==2.9.2 +pytest +pytest-cov pytest-twisted -pytest-cov==2.2.1 +pytest-xdist jmespath brotlipy testfixtures diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index 7c1aacd81..ed7bf0be0 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -1,6 +1,7 @@ -pytest==3.6.3 +pytest +pytest-cov pytest-twisted -pytest-cov==2.5.1 +pytest-xdist testfixtures jmespath leveldb; sys_platform != "win32" From 29739989478bb1b10467b036df91326071b1ffbc Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 14:50:02 +0200 Subject: [PATCH 104/140] Add set_environ test --- tests/test_utils_misc/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index fcb7772ab..738120a0b 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,7 +3,7 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ from tests import mock @@ -130,5 +130,12 @@ class UtilsMiscTestCase(unittest.TestCase): with self.assertRaises(ValueError): create_instance(m, None, None) + def test_set_environ(self): + assert os.environ.get('some_test_environ') is None + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') is None + + if __name__ == "__main__": unittest.main() From 6d52708579be05c29c58d6ccc63486f761466d18 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 15:19:23 +0200 Subject: [PATCH 105/140] Add reset case --- tests/test_utils_misc/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index 738120a0b..eeb995e48 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -136,6 +136,12 @@ class UtilsMiscTestCase(unittest.TestCase): assert os.environ.get('some_test_environ') == 'test_value' assert os.environ.get('some_test_environ') is None + os.environ['some_test_environ'] = 'test' + assert os.environ.get('some_test_environ') == 'test' + with set_environ(some_test_environ='test_value'): + assert os.environ.get('some_test_environ') == 'test_value' + assert os.environ.get('some_test_environ') == 'test' + if __name__ == "__main__": unittest.main() From 935387aaea2f0bdc28504b40dfc9ccbbb437bec4 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Thu, 18 Apr 2019 22:10:23 +0200 Subject: [PATCH 106/140] Revert some non-changes --- scrapy/utils/misc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index cdd5a11c9..f51012e3d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -41,7 +41,7 @@ def load_object(path): except ValueError: raise ValueError("Error loading object '%s': not a full path" % path) - module, name = path[:dot], path[dot + 1:] + module, name = path[:dot], path[dot+1:] mod = import_module(module) try: @@ -86,9 +86,9 @@ def extract_regex(regex, text, encoding='utf-8'): regex = re.compile(regex, re.UNICODE) try: - strings = [regex.search(text).group('extract')] # named group + strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): From 7809c0b14e3ad62aea8e62c7309997ecb64fbbf1 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Sat, 20 Apr 2019 09:25:01 +0200 Subject: [PATCH 107/140] Revert another non-change comment --- scrapy/utils/misc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index f51012e3d..b2164d4a8 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -88,7 +88,7 @@ def extract_regex(regex, text, encoding='utf-8'): try: strings = [regex.search(text).group('extract')] # named group except Exception: - strings = regex.findall(text) # full regex or numbered groups + strings = regex.findall(text) # full regex or numbered groups strings = flatten(strings) if isinstance(text, six.text_type): From 122ca6211935039825aff900e98bc1fbdb4dc0d6 Mon Sep 17 00:00:00 2001 From: Vandenn Date: Thu, 2 May 2019 23:59:01 +0800 Subject: [PATCH 108/140] doc: update configure_logging docs to discourage use with CrawlerProcess --- docs/topics/logging.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 8e280d929..dea0528db 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -238,9 +238,10 @@ scrapy.utils.log module .. autofunction:: configure_logging - ``configure_logging`` is automatically called when using Scrapy commands, - but needs to be called explicitly when running custom scripts. In that - case, its usage is not required but it's recommended. + ``configure_logging`` is automatically called when using Scrapy commands + or :class:`~scrapy.crawler.CrawlerProcess`, but needs to be called explicitly + when running custom scripts using :class:`~scrapy.crawler.CrawlerRunner`. + In that case, its usage is not required but it's recommended. If you plan on configuring the handlers yourself is still recommended you call this function, passing ``install_root_handler=False``. Bear in mind From 8bd207a2f639216eb51f61ae312dfd22f4b39781 Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Sun, 28 Apr 2019 21:47:47 +0200 Subject: [PATCH 109/140] Add documentation --- docs/topics/contracts.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 70f20d4ed..3aa32cba8 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -120,3 +120,22 @@ get the failures pretty printed:: for header in self.args: if header not in response.headers: raise ContractFail('X-CustomHeader not present') + + +Detecting check run +=================== +It is not encouraged to created different behaviour when running test. +However, sometimes it is useful to know when a spider is started by scrapy check. +It can for example be needed to enforce less settings to be set, or to disable some +uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment +variable is set. This can be retrieved via ``os.environ``:: + + import os + import scrapy + + class ExampleSpider(scrapy.Spider): + name = 'example' + + def __init__(self): + if os.environ.get('SCRAPY_CHECK'): + # Do some scraper adjustments when check is running \ No newline at end of file From f6485e669772a940c3c319c71dbcca7bd747d57a Mon Sep 17 00:00:00 2001 From: Matthijs Vos Date: Fri, 3 May 2019 13:53:45 +0200 Subject: [PATCH 110/140] Restore alphabetic order and two lines between import and code --- scrapy/utils/misc.py | 1 + tests/test_utils_misc/__init__.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b2164d4a8..f638adb25 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -12,6 +12,7 @@ from w3lib.html import replace_entities from scrapy.utils.python import flatten, to_unicode from scrapy.item import BaseItem + _ITERABLE_SINGLE_VALUES = dict, BaseItem, six.text_type, bytes diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py index eeb995e48..e109d5343 100644 --- a/tests/test_utils_misc/__init__.py +++ b/tests/test_utils_misc/__init__.py @@ -3,12 +3,13 @@ import os import unittest from scrapy.item import Item, Field -from scrapy.utils.misc import arg_to_iter, create_instance, load_object, walk_modules, set_environ +from scrapy.utils.misc import arg_to_iter, create_instance, load_object, set_environ, walk_modules from tests import mock __doctests__ = ['scrapy.utils.misc'] + class UtilsMiscTestCase(unittest.TestCase): def test_load_object(self): From bc1a92921364de8f4616feb0ea7dcb7b6d42d2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 3 May 2019 14:42:12 +0200 Subject: [PATCH 111/140] Improve the documentation about detecting check runs --- docs/topics/contracts.rst | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 3aa32cba8..9337375bb 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -122,13 +122,12 @@ get the failures pretty printed:: raise ContractFail('X-CustomHeader not present') -Detecting check run -=================== -It is not encouraged to created different behaviour when running test. -However, sometimes it is useful to know when a spider is started by scrapy check. -It can for example be needed to enforce less settings to be set, or to disable some -uploading of result data. When scrapy check is runned the ``SCRAPY_CHECK`` environment -variable is set. This can be retrieved via ``os.environ``:: +Detecting check runs +==================== + +When ``scrapy check`` is running, the ``SCRAPY_CHECK`` environment variable is +set to the ``true`` string. You can use `os.environ`_ to perform any change to +your spiders or your settings when ``scrapy check`` is used:: import os import scrapy @@ -138,4 +137,6 @@ variable is set. This can be retrieved via ``os.environ``:: def __init__(self): if os.environ.get('SCRAPY_CHECK'): - # Do some scraper adjustments when check is running \ No newline at end of file + pass # Do some scraper adjustments when a check is running + +.. _os.environ: https://docs.python.org/3/library/os.html#os.environ From 5814344adfc315a63f43237f02ed75db52765b7d Mon Sep 17 00:00:00 2001 From: Jeffallan <23423962+Jeffallan@users.noreply.github.com> Date: Sat, 4 May 2019 14:15:47 -0500 Subject: [PATCH 112/140] Update telnetconsole.rst Change spelling of bellow to below. --- docs/topics/telnetconsole.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index bf2ffa443..1eb705f05 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -45,7 +45,7 @@ the console you need to type:: >>> By default Username is ``scrapy`` and Password is autogenerated. The -autogenerated Password can be seen on scrapy logs like the example bellow:: +autogenerated Password can be seen on scrapy logs like the example below:: 2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326 From 3a7850fa158148e6c6096add09b555e46949bd51 Mon Sep 17 00:00:00 2001 From: Aditya Date: Sun, 5 May 2019 18:45:40 +0530 Subject: [PATCH 113/140] Update contributing.rst --- docs/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index 2fbe30a00..51b5da59e 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -262,7 +262,7 @@ And their unit-tests are in:: .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://www.python.org/dev/peps/pep-0257/ -.. _pull request: https://help.github.com/send-pull-requests/ +.. _pull request: https://help.github.com/en/articles/creating-a-pull-request .. _pytest: https://docs.pytest.org/en/latest/usage.html .. _pytest-xdist: https://docs.pytest.org/en/3.0.0/xdist.html .. _tox: https://pypi.python.org/pypi/tox From 611249bb7f3a7bb5a92a67d90d6c97f17494768f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 8 May 2019 12:52:29 +0200 Subject: [PATCH 114/140] Implement the METAREFRESH_IGNORE_TAGS setting --- docs/topics/downloader-middleware.rst | 10 ++++++++++ scrapy/downloadermiddlewares/redirect.py | 4 +++- scrapy/settings/default_settings.py | 1 + scrapy/utils/response.py | 4 ++-- tests/test_downloadermiddleware_redirect.py | 19 +++++++++++++++++++ 5 files changed, 35 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f2f3ef466..fa65f66ed 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -805,6 +805,7 @@ The :class:`MetaRefreshMiddleware` can be configured through the following settings (see the settings documentation for more info): * :setting:`METAREFRESH_ENABLED` +* :setting:`METAREFRESH_IGNORE_TAGS` * :setting:`METAREFRESH_MAXDELAY` This middleware obey :setting:`REDIRECT_MAX_TIMES` setting, :reqmeta:`dont_redirect`, @@ -826,6 +827,15 @@ Default: ``True`` Whether the Meta Refresh middleware will be enabled. +.. setting:: METAREFRESH_IGNORE_TAGS + +METAREFRESH_IGNORE_TAGS +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``['script', 'noscript']`` + +Meta tags within these tags are ignored. + .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index cb59d3fd2..49468a2e4 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -88,6 +88,7 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): def __init__(self, settings): super(MetaRefreshMiddleware, self).__init__(settings) + self._ignore_tags = settings.getlist('METAREFRESH_IGNORE_TAGS') self._maxdelay = settings.getint('REDIRECT_MAX_METAREFRESH_DELAY', settings.getint('METAREFRESH_MAXDELAY')) @@ -96,7 +97,8 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware): not isinstance(response, HtmlResponse): return response - interval, url = get_meta_refresh(response) + interval, url = get_meta_refresh(response, + ignore_tags=self._ignore_tags) if url and interval < self._maxdelay: redirected = self._redirect_request_using_get(request, url) return self._redirect(redirected, request, spider, 'meta refresh') diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9986827d8..1ce1516e5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -221,6 +221,7 @@ MEMUSAGE_NOTIFY_MAIL = [] MEMUSAGE_WARNING_MB = 0 METAREFRESH_ENABLED = True +METAREFRESH_IGNORE_TAGS = ['script', 'noscript'] METAREFRESH_MAXDELAY = 100 NEWSPIDER_MODULE = '' diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index bf276b5ca..122af28b0 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -31,12 +31,12 @@ def get_base_url(response): _metaref_cache = weakref.WeakKeyDictionary() -def get_meta_refresh(response): +def get_meta_refresh(response, ignore_tags=('script', 'noscript')): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.text[0:4096] _metaref_cache[response] = html.get_meta_refresh(text, response.url, - response.encoding, ignore_tags=('script', 'noscript')) + response.encoding, ignore_tags=ignore_tags) return _metaref_cache[response] diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 6c81c94ca..0e841489d 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -279,5 +279,24 @@ class MetaRefreshMiddlewareTest(unittest.TestCase): self.assertEqual(req2.meta['redirect_reasons'], ['meta refresh']) self.assertEqual(req3.meta['redirect_reasons'], ['meta refresh', 'meta refresh']) + def test_ignore_tags_default(self): + req = Request(url='http://example.org') + body = ('''''') + rsp = HtmlResponse(req.url, body=body.encode()) + response = self.mw.process_response(req, rsp, self.spider) + assert isinstance(response, Response) + + def test_ignore_tags_empty_list(self): + crawler = get_crawler(Spider, {'METAREFRESH_IGNORE_TAGS': []}) + mw = MetaRefreshMiddleware.from_crawler(crawler) + req = Request(url='http://example.org') + body = ('''''') + rsp = HtmlResponse(req.url, body=body.encode()) + req2 = mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, 'http://example.org/newpage') + if __name__ == "__main__": unittest.main() From e667ca76820a53ac3abf34604fc284761f936bb9 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Fri, 24 May 2019 21:45:53 +0900 Subject: [PATCH 115/140] Account for mangling when serializing requests with private callbacks --- scrapy/utils/reqser.py | 6 +++++- tests/test_utils_reqser.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 959dddbd5..8c99763cf 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -75,7 +75,11 @@ def _find_method(obj, func): pass else: if func_self is obj: - return six.get_method_function(func).__name__ + name = six.get_method_function(func).__name__ + if name.startswith('__'): + classname = obj.__class__.__name__.lstrip('_') + name = '_%s%s' % (classname, name) + return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index dcc070b8f..f7191fcef 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -68,6 +68,12 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_serialization(self): + r = Request("http://www.example.com", + callback=self.spider._TestSpider__parse_item_private, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) @@ -87,6 +93,9 @@ class TestSpider(Spider): def handle_error(self, failure): pass + def __parse_item_private(self, response): + pass + class CustomRequest(Request): pass From 7d36fa7435d2147c7dfd6a87733187823431b61c Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Fri, 24 May 2019 10:32:55 -0300 Subject: [PATCH 116/140] Fix documentation for spiderloader --- docs/topics/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index ba832ab5d..e1623287d 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -154,7 +154,7 @@ Settings API SpiderLoader API ================ -.. module:: scrapy.loader +.. module:: scrapy.spiderloader :synopsis: The spider loader .. class:: SpiderLoader From 0ee2284fcc23ccb2b8a4da8fb561a232cc328fe0 Mon Sep 17 00:00:00 2001 From: Capi Etheriel Date: Fri, 24 May 2019 11:11:15 -0300 Subject: [PATCH 117/140] Add 429 to RETRY_HTTP_CODES --- scrapy/settings/default_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9986827d8..2afa7b321 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -238,7 +238,7 @@ REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408] +RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False From 144afcee7973ab97d6c8d89fec007046cc878e3d Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Sat, 25 May 2019 00:52:00 +0900 Subject: [PATCH 118/140] Use regex to check for private methods --- scrapy/utils/reqser.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 8c99763cf..07c51aaff 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -2,12 +2,16 @@ Helper functions for serializing (and deserializing) requests. """ import six +import re from scrapy.http import Request from scrapy.utils.python import to_unicode, to_native_str from scrapy.utils.misc import load_object +private_name_regex = re.compile('^__[^_](.*[^_])?_?$') + + def request_to_dict(request, spider=None): """Convert Request object to a dict. @@ -76,7 +80,7 @@ def _find_method(obj, func): else: if func_self is obj: name = six.get_method_function(func).__name__ - if name.startswith('__'): + if private_name_regex.search(name): classname = obj.__class__.__name__.lstrip('_') name = '_%s%s' % (classname, name) return name From 18f01ea6ecf1dba77b25d8d1f62c80ed0f9a13f5 Mon Sep 17 00:00:00 2001 From: mar-heaven <775650117@qq.com> Date: Mon, 27 May 2019 17:15:30 +0800 Subject: [PATCH 119/140] remove a "is" When I translated in Chinese, I found a needless "is" --- docs/topics/spiders.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8c4049f85..79eecfc3e 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -657,7 +657,7 @@ SitemapSpider .. attribute:: sitemap_follow - A list of regexes of sitemap that should be followed. This is is only + A list of regexes of sitemap that should be followed. This is only for sites that use `Sitemap index files`_ that point to other sitemap files. From 72b7d3e90ac2d21ffdd0c44878ec1a5a5d0fa5ce Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Mon, 27 May 2019 23:30:23 +0900 Subject: [PATCH 120/140] Make the regex align to the spec better; add unit tests for name variations --- scrapy/utils/reqser.py | 2 +- tests/test_utils_reqser.py | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 07c51aaff..04665a2d4 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -9,7 +9,7 @@ from scrapy.utils.python import to_unicode, to_native_str from scrapy.utils.misc import load_object -private_name_regex = re.compile('^__[^_](.*[^_])?_?$') +private_name_regex = re.compile('^__.*[^_]_?$') def request_to_dict(request, spider=None): diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index f7191fcef..b49450ac5 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -3,7 +3,7 @@ import unittest from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict +from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex class RequestSerializationTest(unittest.TestCase): @@ -74,6 +74,28 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_name_matching(self): + self.assertTrue(private_name_regex.search('__a')) + self.assertTrue(private_name_regex.search('__a_')) + self.assertTrue(private_name_regex.search('__a_a')) + self.assertTrue(private_name_regex.search('__a_a_')) + self.assertTrue(private_name_regex.search('__a__a')) + self.assertTrue(private_name_regex.search('__a__a_')) + self.assertTrue(private_name_regex.search('__a___a')) + self.assertTrue(private_name_regex.search('__a___a_')) + self.assertTrue(private_name_regex.search('___a')) + self.assertTrue(private_name_regex.search('___a_')) + self.assertTrue(private_name_regex.search('___a_a')) + self.assertTrue(private_name_regex.search('___a_a_')) + self.assertTrue(private_name_regex.search('____a_a_')) + + self.assertFalse(private_name_regex.search('_a')) + self.assertFalse(private_name_regex.search('_a_')) + self.assertFalse(private_name_regex.search('__a__')) + self.assertFalse(private_name_regex.search('__')) + self.assertFalse(private_name_regex.search('___')) + self.assertFalse(private_name_regex.search('____')) + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) From 9af91a26b035a10e9303227ad9ddd5e043725514 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Tue, 28 May 2019 01:40:26 +0900 Subject: [PATCH 121/140] Replace regex usage --- scrapy/utils/reqser.py | 10 +++++----- tests/test_utils_reqser.py | 40 +++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 04665a2d4..40223661f 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -2,16 +2,12 @@ Helper functions for serializing (and deserializing) requests. """ import six -import re from scrapy.http import Request from scrapy.utils.python import to_unicode, to_native_str from scrapy.utils.misc import load_object -private_name_regex = re.compile('^__.*[^_]_?$') - - def request_to_dict(request, spider=None): """Convert Request object to a dict. @@ -71,6 +67,10 @@ def request_from_dict(d, spider=None): flags=d.get('flags')) +def _is_private_method(name): + return name.startswith('__') and not name.endswith('__') + + def _find_method(obj, func): if obj: try: @@ -80,7 +80,7 @@ def _find_method(obj, func): else: if func_self is obj: name = six.get_method_function(func).__name__ - if private_name_regex.search(name): + if _is_private_method(name): classname = obj.__class__.__name__.lstrip('_') name = '_%s%s' % (classname, name) return name diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index b49450ac5..fad5b6003 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -3,7 +3,7 @@ import unittest from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict, private_name_regex +from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method class RequestSerializationTest(unittest.TestCase): @@ -75,26 +75,26 @@ class RequestSerializationTest(unittest.TestCase): self._assert_serializes_ok(r, spider=self.spider) def test_private_callback_name_matching(self): - self.assertTrue(private_name_regex.search('__a')) - self.assertTrue(private_name_regex.search('__a_')) - self.assertTrue(private_name_regex.search('__a_a')) - self.assertTrue(private_name_regex.search('__a_a_')) - self.assertTrue(private_name_regex.search('__a__a')) - self.assertTrue(private_name_regex.search('__a__a_')) - self.assertTrue(private_name_regex.search('__a___a')) - self.assertTrue(private_name_regex.search('__a___a_')) - self.assertTrue(private_name_regex.search('___a')) - self.assertTrue(private_name_regex.search('___a_')) - self.assertTrue(private_name_regex.search('___a_a')) - self.assertTrue(private_name_regex.search('___a_a_')) - self.assertTrue(private_name_regex.search('____a_a_')) + self.assertTrue(_is_private_method('__a')) + self.assertTrue(_is_private_method('__a_')) + self.assertTrue(_is_private_method('__a_a')) + self.assertTrue(_is_private_method('__a_a_')) + self.assertTrue(_is_private_method('__a__a')) + self.assertTrue(_is_private_method('__a__a_')) + self.assertTrue(_is_private_method('__a___a')) + self.assertTrue(_is_private_method('__a___a_')) + self.assertTrue(_is_private_method('___a')) + self.assertTrue(_is_private_method('___a_')) + self.assertTrue(_is_private_method('___a_a')) + self.assertTrue(_is_private_method('___a_a_')) + self.assertTrue(_is_private_method('____a_a_')) - self.assertFalse(private_name_regex.search('_a')) - self.assertFalse(private_name_regex.search('_a_')) - self.assertFalse(private_name_regex.search('__a__')) - self.assertFalse(private_name_regex.search('__')) - self.assertFalse(private_name_regex.search('___')) - self.assertFalse(private_name_regex.search('____')) + self.assertFalse(_is_private_method('_a')) + self.assertFalse(_is_private_method('_a_')) + self.assertFalse(_is_private_method('__a__')) + self.assertFalse(_is_private_method('__')) + self.assertFalse(_is_private_method('___')) + self.assertFalse(_is_private_method('____')) def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) From bcad8947e8192448ab3bd59489444efb567f8793 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Mon, 3 Jun 2019 20:41:02 +0900 Subject: [PATCH 122/140] Support inherited private method names --- scrapy/utils/reqser.py | 9 +++++++-- tests/test_utils_reqser.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index 40223661f..d1f472e6e 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -81,8 +81,13 @@ def _find_method(obj, func): if func_self is obj: name = six.get_method_function(func).__name__ if _is_private_method(name): - classname = obj.__class__.__name__.lstrip('_') - name = '_%s%s' % (classname, name) + qualname = getattr(func, '__qualname__', None) + if qualname is None: + classname = obj.__class__.__name__.lstrip('_') + name = '_%s%s' % (classname, name) + else: + splits = qualname.split('.') + name = '_%s%s' % (splits[-2], splits[-1]) return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index fad5b6003..31577bc8c 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import unittest +import sys from scrapy.http import Request, FormRequest from scrapy.spiders import Spider @@ -74,6 +75,14 @@ class RequestSerializationTest(unittest.TestCase): errback=self.spider.handle_error) self._assert_serializes_ok(r, spider=self.spider) + def test_mixin_private_callback_serialization(self): + if sys.version_info[0] < 3: + return + r = Request("http://www.example.com", + callback=self.spider._TestSpiderMixin__mixin_callback, + errback=self.spider.handle_error) + self._assert_serializes_ok(r, spider=self.spider) + def test_private_callback_name_matching(self): self.assertTrue(_is_private_method('__a')) self.assertTrue(_is_private_method('__a_')) @@ -106,7 +115,12 @@ class RequestSerializationTest(unittest.TestCase): self.assertRaises(ValueError, request_to_dict, r) -class TestSpider(Spider): +class TestSpiderMixin(object): + def __mixin_callback(self, response): + pass + + +class TestSpider(Spider, TestSpiderMixin): name = 'test' def parse_item(self, response): From ea209a0ea7815f68a04a5ccab79a2b4f4a146647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 3 Jun 2019 19:21:40 +0200 Subject: [PATCH 123/140] Fix module double indexing issues in the documentation --- docs/topics/stats.rst | 3 +-- docs/topics/telnetconsole.rst | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index dd0c6216b..38648ec55 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -75,8 +75,7 @@ available in Scrapy which extend the basic Stats Collector. You can select which Stats Collector to use through the :setting:`STATS_CLASS` setting. The default Stats Collector used is the :class:`MemoryStatsCollector`. -.. module:: scrapy.statscollectors - :synopsis: Stats Collectors +.. currentmodule:: scrapy.statscollectors MemoryStatsCollector -------------------- diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst index 1eb705f05..7db7e4f6b 100644 --- a/docs/topics/telnetconsole.rst +++ b/docs/topics/telnetconsole.rst @@ -1,12 +1,11 @@ +.. currentmodule:: scrapy.extensions.telnet + .. _topics-telnetconsole: ============== Telnet Console ============== -.. module:: scrapy.extensions.telnet - :synopsis: The Telnet Console - Scrapy comes with a built-in telnet console for inspecting and controlling a Scrapy running process. The telnet console is just a regular python shell running inside the Scrapy process, so you can do literally anything from it. From c7b5ad0e20dc736a8a08b62134c9418439fb7077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 4 Jan 2019 18:17:35 +0100 Subject: [PATCH 124/140] Add a Sphinx extension to generate documentation coverage information --- docs/Makefile | 3 +++ docs/conf.py | 9 ++++++++- docs/contributing.rst | 9 +++++++++ docs/requirements.txt | 2 +- tox.ini | 6 ++++++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 187f03c4c..ff68bf1ae 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -82,6 +82,9 @@ pydoc-topics: build @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ "into the Lib/ directory" +coverage: BUILDER = coverage +coverage: build + htmlview: html $(PYTHON) -c "import webbrowser, os; webbrowser.open('file://' + \ os.path.realpath('build/html/index.html'))" diff --git a/docs/conf.py b/docs/conf.py index a54a6bbe9..832626f6b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,8 @@ sys.path.insert(0, path.dirname(path.dirname(__file__))) # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ 'scrapydocs', - 'sphinx.ext.autodoc' + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', ] # Add any paths that contain templates here, relative to this directory. @@ -218,3 +219,9 @@ linkcheck_ignore = [ 'http://localhost:\d+', 'http://hg.scrapy.org', 'http://directory.google.com/' ] + + +# Options for the Coverage extension +# ---------------------------------- +coverage_ignore_pyobjects = [ +] diff --git a/docs/contributing.rst b/docs/contributing.rst index 51b5da59e..b4f91ea8d 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -99,6 +99,15 @@ Well-written patches should: the documentation changes in the same patch. See `Documentation policies`_ below. +* if you're adding a private API, please add a regular expression to the + ``coverage_ignore_pyobjects`` variable of ``docs/conf.py`` to exclude the new + private API from documentation coverage checks. + + To see if your private API is skipped properly, generate a documentation + coverage report as follows:: + + tox -e docs-coverage + .. _submitting-patches: Submitting patches diff --git a/docs/requirements.txt b/docs/requirements.txt index 8e7611d21..379da9994 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -Sphinx>=1.6 +Sphinx>=2.1 sphinx_rtd_theme \ No newline at end of file diff --git a/tox.ini b/tox.ini index 0c0f8f7b7..157a8b3ed 100644 --- a/tox.ini +++ b/tox.ini @@ -105,6 +105,12 @@ deps = {[docs]deps} commands = sphinx-build -W -b html . {envtmpdir}/html +[testenv:docs-coverage] +changedir = {[docs]changedir} +deps = {[docs]deps} +commands = + sphinx-build -b coverage . {envtmpdir}/coverage + [testenv:docs-links] changedir = {[docs]changedir} deps = {[docs]deps} From c81e15ed6ede552c499ae3ac4e03af27b1f9ed89 Mon Sep 17 00:00:00 2001 From: Artem Kuchumov Date: Wed, 5 Jun 2019 13:15:23 +0500 Subject: [PATCH 125/140] Tutorial: scrapy shell example should say "text" not "title" (#3807) Tutorial: scrapy shell example should say "text" not "title" --- docs/intro/tutorial.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8bd2d27dd..a190ce407 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -379,11 +379,11 @@ variable, so that we can run our CSS selectors directly on a particular quote:: >>> quote = response.css("div.quote")[0] -Now, let's extract ``title``, ``author`` and the ``tags`` from that quote +Now, let's extract ``text``, ``author`` and the ``tags`` from that quote using the ``quote`` object we just created:: - >>> title = quote.css("span.text::text").get() - >>> title + >>> text = quote.css("span.text::text").get() + >>> text '“The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.”' >>> author = quote.css("small.author::text").get() >>> author From 9c81721c407ff41ef9dce2c33e26ac477355cf1f Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Wed, 5 Jun 2019 23:43:56 +0900 Subject: [PATCH 126/140] Add tests for private method name mangling --- scrapy/utils/reqser.py | 18 +++++++++++------- tests/test_utils_reqser.py | 16 +++++++++++++++- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/scrapy/utils/reqser.py b/scrapy/utils/reqser.py index d1f472e6e..3c463cfed 100644 --- a/scrapy/utils/reqser.py +++ b/scrapy/utils/reqser.py @@ -71,6 +71,16 @@ def _is_private_method(name): return name.startswith('__') and not name.endswith('__') +def _mangle_private_name(obj, func, name): + qualname = getattr(func, '__qualname__', None) + if qualname is None: + classname = obj.__class__.__name__.lstrip('_') + return '_%s%s' % (classname, name) + else: + splits = qualname.split('.') + return '_%s%s' % (splits[-2], splits[-1]) + + def _find_method(obj, func): if obj: try: @@ -81,13 +91,7 @@ def _find_method(obj, func): if func_self is obj: name = six.get_method_function(func).__name__ if _is_private_method(name): - qualname = getattr(func, '__qualname__', None) - if qualname is None: - classname = obj.__class__.__name__.lstrip('_') - name = '_%s%s' % (classname, name) - else: - splits = qualname.split('.') - name = '_%s%s' % (splits[-2], splits[-1]) + return _mangle_private_name(obj, func, name) return name raise ValueError("Function %s is not a method of: %s" % (func, obj)) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 31577bc8c..7f9e31daa 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -2,9 +2,11 @@ import unittest import sys +import six + from scrapy.http import Request, FormRequest from scrapy.spiders import Spider -from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method +from scrapy.utils.reqser import request_to_dict, request_from_dict, _is_private_method, _mangle_private_name class RequestSerializationTest(unittest.TestCase): @@ -105,6 +107,18 @@ class RequestSerializationTest(unittest.TestCase): self.assertFalse(_is_private_method('___')) self.assertFalse(_is_private_method('____')) + def _assert_mangles_to(self, obj, name): + self.assertEqual( + _mangle_private_name(obj, getattr(obj, name), name), + name + ) + + def test_private_name_mangling(self): + self._assert_mangles_to( + self.spider, '_TestSpider__parse_item_private') + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') + def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) self.assertRaises(ValueError, request_to_dict, r) From 3dd3e8c29863683d60f9c4f74aacac3103703061 Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Wed, 5 Jun 2019 23:49:54 +0900 Subject: [PATCH 127/140] Restrict different class mangling tests to Py 3+ --- tests/test_utils_reqser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 7f9e31daa..57dc5db53 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -116,8 +116,9 @@ class RequestSerializationTest(unittest.TestCase): def test_private_name_mangling(self): self._assert_mangles_to( self.spider, '_TestSpider__parse_item_private') - self._assert_mangles_to( - self.spider, '_TestSpiderMixin__mixin_callback') + if sys.version_info[0] >= 3: + self._assert_mangles_to( + self.spider, '_TestSpiderMixin__mixin_callback') def test_unserializable_callback1(self): r = Request("http://www.example.com", callback=lambda x: x) From 6af1dc89aa5988ebbfbef90afdafa84736f3993c Mon Sep 17 00:00:00 2001 From: Andrew Baxter Date: Thu, 6 Jun 2019 04:25:19 +0900 Subject: [PATCH 128/140] Fix mangling test --- tests/test_utils_reqser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_utils_reqser.py b/tests/test_utils_reqser.py index 57dc5db53..e5a09dcf1 100644 --- a/tests/test_utils_reqser.py +++ b/tests/test_utils_reqser.py @@ -108,8 +108,9 @@ class RequestSerializationTest(unittest.TestCase): self.assertFalse(_is_private_method('____')) def _assert_mangles_to(self, obj, name): + func = getattr(obj, name) self.assertEqual( - _mangle_private_name(obj, getattr(obj, name), name), + _mangle_private_name(obj, func, func.__name__), name ) From bd8a10384b462dd56b33668e8b92e4a148fd6fba Mon Sep 17 00:00:00 2001 From: Sortafreel Date: Fri, 7 Jun 2019 01:50:03 +0300 Subject: [PATCH 129/140] Add values (if there're any) when initiating items from dicts https://github.com/scrapy/scrapy/issues/3804 --- scrapy/loader/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index a7c75a46a..295a8e42d 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -35,6 +35,8 @@ class ItemLoader(object): self.parent = parent self._local_item = context['item'] = item self._local_values = defaultdict(list) + for field_name, value in item.items(): + self.add_value(field_name, value) @property def _values(self): From 754f52b02781097c8ca6835e057815c7653062d4 Mon Sep 17 00:00:00 2001 From: Sortafreel Date: Fri, 7 Jun 2019 03:20:45 +0300 Subject: [PATCH 130/140] Preprocess values if item built from dict. https://github.com/scrapy/scrapy/issues/3804 --- scrapy/loader/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 295a8e42d..7c7f66866 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -35,8 +35,9 @@ class ItemLoader(object): self.parent = parent self._local_item = context['item'] = item self._local_values = defaultdict(list) + # Preprocess values if item built from dict for field_name, value in item.items(): - self.add_value(field_name, value) + self._values[field_name] = self._process_input_value(field_name, value) @property def _values(self): From c7ba72b5dc9da3435eb1ec303b991d05ba40ba1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 4 Jun 2019 17:10:14 +0200 Subject: [PATCH 131/140] Skip scrapy.contracts private APIs in the documentation coverage report --- docs/conf.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/conf.py b/docs/conf.py index 832626f6b..bf222b361 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -224,4 +224,17 @@ linkcheck_ignore = [ # Options for the Coverage extension # ---------------------------------- coverage_ignore_pyobjects = [ + # Contract’s add_pre_hook and add_post_hook are not documented because + # they should be transparent to contract developers, for whom pre_hook and + # post_hook should be the actual concern. + r'\bContract\.add_(pre|post)_hook$', + + # ContractsManager is an internal class, developers are not expected to + # interact with it directly in any way. + r'\bContractsManager\b$', + + # For default contracts we only want to document their general purpose in + # their constructor, the methods they reimplement to achieve that purpose + # should be irrelevant to developers using those contracts. + r'\w+Contract\.(adjust_request_args|(pre|post)_process)$', ] From a1bca6a8e722af53241e51bbf758e7bd67671801 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 07:36:29 +0300 Subject: [PATCH 132/140] Add tests. --- scrapy/loader/__init__.py | 1 + tests/test_loader.py | 65 ++++++++++++++++++++++++++------------- 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 7c7f66866..20f0f90c3 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -36,6 +36,7 @@ class ItemLoader(object): self._local_item = context['item'] = item self._local_values = defaultdict(list) # Preprocess values if item built from dict + # Values need to be added to item._values if added them from dict (not with add_values) for field_name, value in item.items(): self._values[field_name] = self._process_input_value(field_name, value) diff --git a/tests/test_loader.py b/tests/test_loader.py index 8b58e4dbd..eb4a01572 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -419,6 +419,29 @@ class BasicItemLoaderTest(unittest.TestCase): self.assertEqual(item['url'], u'rabbit.hole') self.assertEqual(item['summary'], u'rabbithole') + def test_create_item_from_dict(self): + class TestItem(Item): + title = Field() + + class TestItemLoader(ItemLoader): + default_item_class = TestItem + + input_item = {'title': 'Test item title 1'} + il = TestItemLoader(item=input_item) + # Getting output value mustn't remove value from item + self.assertEqual(il.load_item(), { + 'title': 'Test item title 1', + }) + self.assertEqual(il.get_output_value('title'), 'Test item title 1') + self.assertEqual(il.load_item(), { + 'title': 'Test item title 1', + }) + + input_item = {'title': 'Test item title 2'} + il = TestItemLoader(item=input_item) + # Values from dict must be added to item _values + self.assertEqual(il._values.get('title'), 'Test item title 2') + class ProcessorsTest(unittest.TestCase): @@ -709,28 +732,28 @@ class SubselectorLoaderTest(unittest.TestCase): class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) + def test_output(self): + for l in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[l] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(l, test, expected) + ) if __name__ == "__main__": From 7dad2f7b130c426f2a8aee320ccbc378752a9568 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 07:43:03 +0300 Subject: [PATCH 133/140] Add more tests. --- tests/test_loader.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_loader.py b/tests/test_loader.py index eb4a01572..241630ab3 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -442,6 +442,20 @@ class BasicItemLoaderTest(unittest.TestCase): # Values from dict must be added to item _values self.assertEqual(il._values.get('title'), 'Test item title 2') + input_item = {'title': [u'Test item title 3', u'Test item 4']} + il = TestItemLoader(item=input_item) + # Same rules must work for lists + self.assertEqual(il._values.get('title'), + [u'Test item title 3', u'Test item 4']) + self.assertEqual(il.load_item(), { + 'title': [u'Test item title 3', u'Test item 4'], + }) + self.assertEqual(il.get_output_value('title'), + [u'Test item title 3', u'Test item 4']) + self.assertEqual(il.load_item(), { + 'title': [u'Test item title 3', u'Test item 4'], + }) + class ProcessorsTest(unittest.TestCase): From 0da972339bb174156b08a3ae34ece7fddea1e48d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Jun 2019 14:11:38 +0200 Subject: [PATCH 134/140] Require Twisted<=19.2.0 for Python 3.4 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bd666e93c..4dc6d18c1 100644 --- a/setup.py +++ b/setup.py @@ -65,7 +65,8 @@ setup( ], python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', install_requires=[ - 'Twisted>=13.1.0', + 'Twisted>=13.1.0;python_version!="3.4"', + 'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"', 'w3lib>=1.17.0', 'queuelib', 'lxml', From fe0f80f2f422d4047a8b6230d66eb853d443d90b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 11 Jun 2019 15:50:41 +0200 Subject: [PATCH 135/140] Set the cloned directory as PYTHONPATH in appveyor.yml --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 93cfd469e..7fd636864 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,7 +12,8 @@ branches: install: - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" - - "SET TOX_TESTENV_PASSENV=HOME USERPROFILE HOMEPATH HOMEDRIVE" + - "SET PYTHONPATH=%APPVEYOR_BUILD_FOLDER%" + - "SET TOX_TESTENV_PASSENV=HOME HOMEDRIVE HOMEPATH PYTHONPATH USERPROFILE" - "pip install -U tox" build: false From cdeccac6d6ccd0034a5f007ed371c1d481b32c26 Mon Sep 17 00:00:00 2001 From: sortafreel Date: Tue, 11 Jun 2019 17:38:06 +0300 Subject: [PATCH 136/140] Linting (return previous indentation). --- tests/test_loader.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_loader.py b/tests/test_loader.py index 241630ab3..5a8ee1b2e 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -746,28 +746,28 @@ class SubselectorLoaderTest(unittest.TestCase): class SelectJmesTestCase(unittest.TestCase): - test_list_equals = { - 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), - 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), - 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), - 'dict': ( - 'foo.bar[*].name', - {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, - ['one', 'two'] - ), - 'list': ('[1]', [1, 2], 2) - } + test_list_equals = { + 'simple': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'invalid': ('foo.bar.baz', {"foo": {"bar": "baz"}}, None), + 'top_level': ('foo', {"foo": {"bar": "baz"}}, {"bar": "baz"}), + 'double_vs_single_quote_string': ('foo.bar', {"foo": {"bar": "baz"}}, "baz"), + 'dict': ( + 'foo.bar[*].name', + {"foo": {"bar": [{"name": "one"}, {"name": "two"}]}}, + ['one', 'two'] + ), + 'list': ('[1]', [1, 2], 2) + } - def test_output(self): - for l in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[l] - test = SelectJmes(expr)(test_list) - self.assertEqual( - test, - expected, - msg='test "{}" got {} expected {}'.format(l, test, expected) - ) + def test_output(self): + for l in self.test_list_equals: + expr, test_list, expected = self.test_list_equals[l] + test = SelectJmes(expr)(test_list) + self.assertEqual( + test, + expected, + msg='test "{}" got {} expected {}'.format(l, test, expected) + ) if __name__ == "__main__": From b8900ec6a698cb4e27424de57ca5593f1c7300e7 Mon Sep 17 00:00:00 2001 From: Anubhav Patel Date: Mon, 17 Jun 2019 00:06:44 +0530 Subject: [PATCH 137/140] removes unused var --- tests/test_downloadermiddleware.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 0f420b70d..03564e748 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -123,7 +123,6 @@ class ProcessRequestInvalidOutput(ManagerTestCase): def test_invalid_process_request(self): req = Request('http://example.com/index.html') - resp = Response('http://example.com/index.html') class InvalidProcessRequestMiddleware: def process_request(self, request, spider): @@ -143,7 +142,6 @@ class ProcessResponseInvalidOutput(ManagerTestCase): def test_invalid_process_response(self): req = Request('http://example.com/index.html') - resp = Response('http://example.com/index.html') class InvalidProcessResponseMiddleware: def process_response(self, request, response, spider): @@ -163,7 +161,6 @@ class ProcessExceptionInvalidOutput(ManagerTestCase): def test_invalid_process_exception(self): req = Request('http://example.com/index.html') - resp = Response('http://example.com/index.html') class InvalidProcessExceptionMiddleware: def process_request(self, request, spider): From f4f2b1695c4d7bc69e5cb19c33a3a47f69bd1e8d Mon Sep 17 00:00:00 2001 From: Victor Torres Date: Mon, 24 Jun 2019 07:38:05 -0300 Subject: [PATCH 138/140] Fix a memory leak on the Media Pipeline (Files and Images) (#3813) We're storing exceptions captured by Twisted on the media pipeline cache, but we're also using the defer.returnValue method with our own methods decorated with @defer.inlineCallbacks. The defer.returnValue method passes returned values forward by throwing a defer._DefGen_Return exception, which in its turn extends the BaseException class and is captured by Twisted. This way, the latest exception stored in the Failure's object may also have an HtmlResponse object in its __context__ attribute. As the Response object also keeps track of the Request object that has originated it, you could figure it out how many RAM we're wasting here. This could easily lead to a Memory Leak problem when running spiders with Media Pipeline enabled and a particular Request set that tends to raise a significant number of exceptions. Example triggers: - media requests with 404 status responses - user land exceptins coming from custom middlewares - etc. --- scrapy/pipelines/media.py | 26 +++++++++++- tests/test_pipeline_media.py | 77 +++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 404bbf5bf..95dca9a3f 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -3,7 +3,7 @@ from __future__ import print_function import functools import logging from collections import defaultdict -from twisted.internet.defer import Deferred, DeferredList +from twisted.internet.defer import Deferred, DeferredList, _DefGen_Return from twisted.python.failure import Failure from scrapy.settings import Settings @@ -139,6 +139,30 @@ class MediaPipeline(object): result.cleanFailure() result.frames = [] result.stack = None + + # This code fixes a memory leak by avoiding to keep references to + # the Request and Response objects on the Media Pipeline cache. + # + # Twisted inline callbacks pass return values using the function + # twisted.internet.defer.returnValue, which encapsulates the return + # value inside a _DefGen_Return base exception. + # + # What happens when the media_downloaded callback raises another + # exception, for example a FileException('download-error') when + # the Response status code is not 200 OK, is that it stores the + # _DefGen_Return exception on the FileException context. + # + # To avoid keeping references to the Response and therefore Request + # objects on the Media Pipeline cache, we should wipe the context of + # the exception encapsulated by the Twisted Failure when its a + # _DefGen_Return instance. + # + # This problem does not occur in Python 2.7 since we don't have + # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). + context = getattr(result.value, '__context__', None) + if isinstance(context, _DefGen_Return): + setattr(result.value, '__context__', None) + info.downloading.remove(fp) info.downloaded[fp] = result # cache result for wad in info.waiting.pop(fp): diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 5f6a6d9e6..28e39cefa 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -1,15 +1,19 @@ from __future__ import print_function + +import sys + from testfixtures import LogCapture from twisted.trial import unittest from twisted.python.failure import Failure from twisted.internet import reactor -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred, inlineCallbacks, returnValue from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.request import request_fingerprint from scrapy.pipelines.media import MediaPipeline +from scrapy.pipelines.files import FileException from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy import signals @@ -90,6 +94,77 @@ class BaseMediaPipelineTestCase(unittest.TestCase): self.pipe._modify_media_request(request) assert request.meta == {'handle_httpstatus_all': True} + def test_should_remove_req_res_references_before_caching_the_results(self): + """Regression test case to prevent a memory leak in the Media Pipeline. + + The memory leak is triggered when an exception is raised when a Response + scheduled by the Media Pipeline is being returned. For example, when a + FileException('download-error') is raised because the Response status + code is not 200 OK. + + It happens because we are keeping a reference to the Response object + inside the FileException context. This is caused by the way Twisted + return values from inline callbacks. It raises a custom exception + encapsulating the original return value. + + The solution is to remove the exception context when this context is a + _DefGen_Return instance, the BaseException used by Twisted to pass the + returned value from those inline callbacks. + + Maybe there's a better and more reliable way to test the case described + here, but it would be more complicated and involve running - or at least + mocking - some async steps from the Media Pipeline. The current test + case is simple and detects the problem very fast. On the other hand, it + would not detect another kind of leak happening due to old object + references being kept inside the Media Pipeline cache. + + This problem does not occur in Python 2.7 since we don't have Exception + Chaining (https://www.python.org/dev/peps/pep-3134/). + """ + # Create sample pair of Request and Response objects + request = Request('http://url') + response = Response('http://url', body=b'', request=request) + + # Simulate the Media Pipeline behavior to produce a Twisted Failure + try: + # Simulate a Twisted inline callback returning a Response + # The returnValue method raises an exception encapsulating the value + returnValue(response) + except BaseException as exc: + def_gen_return_exc = exc + try: + # Simulate the media_downloaded callback raising a FileException + # This usually happens when the status code is not 200 OK + raise FileException('download-error') + except Exception as exc: + file_exc = exc + # Simulate Twisted capturing the FileException + # It encapsulates the exception inside a Twisted Failure + failure = Failure(file_exc) + + # The Failure should encapsulate a FileException ... + self.assertEqual(failure.value, file_exc) + # ... and if we're running on Python 3 ... + if sys.version_info.major >= 3: + # ... it should have the returnValue exception set as its context + self.assertEqual(failure.value.__context__, def_gen_return_exc) + + # Let's calculate the request fingerprint and fake some runtime data... + fp = request_fingerprint(request) + info = self.pipe.spiderinfo + info.downloading.add(fp) + info.waiting[fp] = [] + + # When calling the method that caches the Request's result ... + self.pipe._cache_result_and_execute_waiters(failure, fp, info) + # ... it should store the Twisted Failure ... + self.assertEqual(info.downloaded[fp], failure) + # ... encapsulating the original FileException ... + self.assertEqual(info.downloaded[fp].value, file_exc) + # ... but it should not store the returnValue exception on its context + context = getattr(info.downloaded[fp].value, '__context__', None) + self.assertIsNone(context) + class MockedMediaPipeline(MediaPipeline): From 8a3b15eb91169ab262e4dca60105f56467ecd1ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 27 Mar 2019 08:50:33 +0100 Subject: [PATCH 139/140] Document how to select dynamically-loaded content --- docs/index.rst | 4 + docs/topics/dynamic-content.rst | 246 ++++++++++++++++++++++++++++++++ 2 files changed, 250 insertions(+) create mode 100644 docs/topics/dynamic-content.rst diff --git a/docs/index.rst b/docs/index.rst index cedde8f38..6d5f9e77d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -158,6 +158,7 @@ Solving specific problems topics/practices topics/broad-crawls topics/developer-tools + topics/dynamic-content topics/leaks topics/media-pipeline topics/deploy @@ -183,6 +184,9 @@ Solving specific problems :doc:`topics/developer-tools` Learn how to scrape with your browser's developer tools. +:doc:`topics/dynamic-content` + Read webpage data that is loaded dynamically. + :doc:`topics/leaks` Learn how to find and get rid of memory leaks in your crawler. diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst new file mode 100644 index 000000000..8b5dacf56 --- /dev/null +++ b/docs/topics/dynamic-content.rst @@ -0,0 +1,246 @@ +.. _topics-dynamic-content: + +==================================== +Selecting dynamically-loaded content +==================================== + +Some webpages show the desired data when you load them in a web browser. +However, when you download them using Scrapy, you cannot reach the desired data +using :ref:`selectors `. + +When this happens, the recommended approach is to +:ref:`find the data source ` and extract the data +from it. + +If you fail to do that, and you can nonetheless access the desired data through +the :ref:`DOM ` from your web browser, see +:ref:`topics-javascript-rendering`. + +.. _topics-finding-data-source: + +Finding the data source +======================= + +To extract the desired data, you must first find its source location. + +If the data is in a non-text-based format, such as an image or a PDF document, +use the :ref:`network tool ` of your web browser to find +the corresponding request, and :ref:`reproduce it +`. + +If your web browser lets you select the desired data as text, the data may be +defined in embedded JavaScript code, or loaded from an external resource in a +text-based format. + +In that case, you can use a tool like wgrep_ to find the URL of that resource. + +If the data turns out to come from the original URL itself, you must +:ref:`inspect the source code of the webpage ` to +determine where the data is located. + +If the data comes from a different URL, you will need to :ref:`reproduce the +corresponding request `. + +.. _topics-inspecting-source: + +Inspecting the source code of a webpage +======================================= + +Sometimes you need to inspect the source code of a webpage (not the +:ref:`DOM `) to determine where some desired data is located. + +Use Scrapy’s :command:`fetch` command to download the webpage contents as seen +by Scrapy:: + + scrapy fetch --nolog https://example.com > response.html + +If the desired data is in embedded JavaScript code within a ``